2024-11-24 04:25:28,431 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@20b2475a 2024-11-24 04:25:28,443 main DEBUG Took 0.009586 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-24 04:25:28,443 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-24 04:25:28,443 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-24 04:25:28,444 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-24 04:25:28,445 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 04:25:28,453 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-24 04:25:28,464 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 04:25:28,466 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 04:25:28,466 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 04:25:28,467 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 04:25:28,467 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 04:25:28,467 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 04:25:28,468 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 04:25:28,469 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 04:25:28,469 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 04:25:28,469 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 04:25:28,470 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 04:25:28,470 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 04:25:28,471 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 04:25:28,471 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 04:25:28,471 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 04:25:28,472 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 04:25:28,472 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 04:25:28,472 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 04:25:28,473 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 04:25:28,473 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 04:25:28,473 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 04:25:28,474 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 04:25:28,474 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 04:25:28,474 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 04:25:28,475 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 04:25:28,475 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-24 04:25:28,476 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 04:25:28,478 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-24 04:25:28,479 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-24 04:25:28,480 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-24 04:25:28,481 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-24 04:25:28,481 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-24 04:25:28,489 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-24 04:25:28,491 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-24 04:25:28,493 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-24 04:25:28,493 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-24 04:25:28,494 main DEBUG createAppenders(={Console}) 2024-11-24 04:25:28,494 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@20b2475a initialized 2024-11-24 04:25:28,495 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@20b2475a 2024-11-24 04:25:28,495 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@20b2475a OK. 2024-11-24 04:25:28,495 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-24 04:25:28,495 main DEBUG OutputStream closed 2024-11-24 04:25:28,496 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-24 04:25:28,496 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-24 04:25:28,496 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@4310d43 OK 2024-11-24 04:25:28,565 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-24 04:25:28,567 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-24 04:25:28,568 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-24 04:25:28,569 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-24 04:25:28,570 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-24 04:25:28,570 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-24 04:25:28,570 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-24 04:25:28,571 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-24 04:25:28,571 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-24 04:25:28,571 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-24 04:25:28,572 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-24 04:25:28,572 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-24 04:25:28,572 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-24 04:25:28,572 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-24 04:25:28,573 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-24 04:25:28,573 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-24 04:25:28,573 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-24 04:25:28,574 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-24 04:25:28,576 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-24 04:25:28,577 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@12f9af83) with optional ClassLoader: null 2024-11-24 04:25:28,577 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-24 04:25:28,577 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@12f9af83] started OK. 2024-11-24T04:25:28,831 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6816aaf7-45a5-053e-0edc-4c140671afba 2024-11-24 04:25:28,835 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-24 04:25:28,836 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-24T04:25:28,849 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay timeout: 13 mins 2024-11-24T04:25:28,858 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplayValueCompression timeout: 13 mins 2024-11-24T04:25:28,884 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-24T04:25:28,942 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-24T04:25:28,942 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-24T04:25:28,956 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=3, rsPorts=, rsClass=null, numDataNodes=3, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-24T04:25:28,972 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6816aaf7-45a5-053e-0edc-4c140671afba/cluster_be0b9e06-9b41-befd-1079-43744c8c01cc, deleteOnExit=true 2024-11-24T04:25:28,973 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-24T04:25:28,974 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6816aaf7-45a5-053e-0edc-4c140671afba/test.cache.data in system properties and HBase conf 2024-11-24T04:25:28,974 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6816aaf7-45a5-053e-0edc-4c140671afba/hadoop.tmp.dir in system properties and HBase conf 2024-11-24T04:25:28,975 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6816aaf7-45a5-053e-0edc-4c140671afba/hadoop.log.dir in system properties and HBase conf 2024-11-24T04:25:28,975 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6816aaf7-45a5-053e-0edc-4c140671afba/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-24T04:25:28,975 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6816aaf7-45a5-053e-0edc-4c140671afba/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-24T04:25:28,976 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-24T04:25:29,061 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-24T04:25:29,193 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-24T04:25:29,199 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6816aaf7-45a5-053e-0edc-4c140671afba/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-24T04:25:29,199 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6816aaf7-45a5-053e-0edc-4c140671afba/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-24T04:25:29,200 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6816aaf7-45a5-053e-0edc-4c140671afba/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-24T04:25:29,201 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6816aaf7-45a5-053e-0edc-4c140671afba/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-24T04:25:29,201 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6816aaf7-45a5-053e-0edc-4c140671afba/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-24T04:25:29,202 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6816aaf7-45a5-053e-0edc-4c140671afba/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-24T04:25:29,202 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6816aaf7-45a5-053e-0edc-4c140671afba/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-24T04:25:29,203 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6816aaf7-45a5-053e-0edc-4c140671afba/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-24T04:25:29,203 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6816aaf7-45a5-053e-0edc-4c140671afba/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-24T04:25:29,204 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6816aaf7-45a5-053e-0edc-4c140671afba/nfs.dump.dir in system properties and HBase conf 2024-11-24T04:25:29,205 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6816aaf7-45a5-053e-0edc-4c140671afba/java.io.tmpdir in system properties and HBase conf 2024-11-24T04:25:29,205 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6816aaf7-45a5-053e-0edc-4c140671afba/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-24T04:25:29,206 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6816aaf7-45a5-053e-0edc-4c140671afba/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-24T04:25:29,206 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6816aaf7-45a5-053e-0edc-4c140671afba/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-24T04:25:30,038 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-24T04:25:30,115 INFO [Time-limited test {}] log.Log(170): Logging initialized @2406ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-24T04:25:30,190 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-24T04:25:30,254 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-24T04:25:30,279 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-24T04:25:30,280 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-24T04:25:30,281 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-24T04:25:30,294 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-24T04:25:30,296 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4f37ffca{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6816aaf7-45a5-053e-0edc-4c140671afba/hadoop.log.dir/,AVAILABLE} 2024-11-24T04:25:30,297 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6dc9d5c1{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-24T04:25:30,499 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@3717288f{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6816aaf7-45a5-053e-0edc-4c140671afba/java.io.tmpdir/jetty-localhost-36685-hadoop-hdfs-3_4_1-tests_jar-_-any-15500202954010349828/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-24T04:25:30,505 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@4bd70930{HTTP/1.1, (http/1.1)}{localhost:36685} 2024-11-24T04:25:30,505 INFO [Time-limited test {}] server.Server(415): Started @2796ms 2024-11-24T04:25:30,937 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-24T04:25:30,947 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-24T04:25:30,949 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-24T04:25:30,949 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-24T04:25:30,949 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-24T04:25:30,965 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@cf5a85e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6816aaf7-45a5-053e-0edc-4c140671afba/hadoop.log.dir/,AVAILABLE} 2024-11-24T04:25:30,966 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4a359997{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-24T04:25:31,135 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@330740de{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6816aaf7-45a5-053e-0edc-4c140671afba/java.io.tmpdir/jetty-localhost-32989-hadoop-hdfs-3_4_1-tests_jar-_-any-14463895811888948228/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-24T04:25:31,136 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@7b24cab9{HTTP/1.1, (http/1.1)}{localhost:32989} 2024-11-24T04:25:31,136 INFO [Time-limited test {}] server.Server(415): Started @3427ms 2024-11-24T04:25:31,193 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-24T04:25:31,336 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-24T04:25:31,341 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-24T04:25:31,344 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-24T04:25:31,344 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-24T04:25:31,344 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-24T04:25:31,346 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@46b092e1{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6816aaf7-45a5-053e-0edc-4c140671afba/hadoop.log.dir/,AVAILABLE} 2024-11-24T04:25:31,346 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@5cc2d6b{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-24T04:25:31,465 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@7bd427b8{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6816aaf7-45a5-053e-0edc-4c140671afba/java.io.tmpdir/jetty-localhost-44967-hadoop-hdfs-3_4_1-tests_jar-_-any-3474150685792149211/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-24T04:25:31,466 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@6915083f{HTTP/1.1, (http/1.1)}{localhost:44967} 2024-11-24T04:25:31,466 INFO [Time-limited test {}] server.Server(415): Started @3757ms 2024-11-24T04:25:31,468 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-24T04:25:31,506 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-24T04:25:31,511 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-24T04:25:31,513 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-24T04:25:31,513 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-24T04:25:31,513 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-24T04:25:31,514 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@2a6d5e13{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6816aaf7-45a5-053e-0edc-4c140671afba/hadoop.log.dir/,AVAILABLE} 2024-11-24T04:25:31,515 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@f9972d0{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-24T04:25:31,656 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@35f1150e{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6816aaf7-45a5-053e-0edc-4c140671afba/java.io.tmpdir/jetty-localhost-45039-hadoop-hdfs-3_4_1-tests_jar-_-any-1626755171008126392/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-24T04:25:31,657 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@13a77e13{HTTP/1.1, (http/1.1)}{localhost:45039} 2024-11-24T04:25:31,657 INFO [Time-limited test {}] server.Server(415): Started @3948ms 2024-11-24T04:25:31,659 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-24T04:25:31,695 WARN [Thread-119 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6816aaf7-45a5-053e-0edc-4c140671afba/cluster_be0b9e06-9b41-befd-1079-43744c8c01cc/data/data2/current/BP-1091946017-172.17.0.2-1732422329789/current, will proceed with Du for space computation calculation, 2024-11-24T04:25:31,695 WARN [Thread-118 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6816aaf7-45a5-053e-0edc-4c140671afba/cluster_be0b9e06-9b41-befd-1079-43744c8c01cc/data/data3/current/BP-1091946017-172.17.0.2-1732422329789/current, will proceed with Du for space computation calculation, 2024-11-24T04:25:31,695 WARN [Thread-117 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6816aaf7-45a5-053e-0edc-4c140671afba/cluster_be0b9e06-9b41-befd-1079-43744c8c01cc/data/data1/current/BP-1091946017-172.17.0.2-1732422329789/current, will proceed with Du for space computation calculation, 2024-11-24T04:25:31,695 WARN [Thread-120 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6816aaf7-45a5-053e-0edc-4c140671afba/cluster_be0b9e06-9b41-befd-1079-43744c8c01cc/data/data4/current/BP-1091946017-172.17.0.2-1732422329789/current, will proceed with Du for space computation calculation, 2024-11-24T04:25:31,747 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-24T04:25:31,747 WARN [Thread-81 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-24T04:25:31,800 WARN [Thread-139 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6816aaf7-45a5-053e-0edc-4c140671afba/cluster_be0b9e06-9b41-befd-1079-43744c8c01cc/data/data5/current/BP-1091946017-172.17.0.2-1732422329789/current, will proceed with Du for space computation calculation, 2024-11-24T04:25:31,801 WARN [Thread-140 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6816aaf7-45a5-053e-0edc-4c140671afba/cluster_be0b9e06-9b41-befd-1079-43744c8c01cc/data/data6/current/BP-1091946017-172.17.0.2-1732422329789/current, will proceed with Du for space computation calculation, 2024-11-24T04:25:31,829 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x1600f6512f5f0072 with lease ID 0xaa9cca750b8583da: Processing first storage report for DS-20bb5930-23a3-441d-bdb2-015c0d56b2ec from datanode DatanodeRegistration(127.0.0.1:34457, datanodeUuid=c4648929-9bd7-4a94-827d-36f0dcfe9f17, infoPort=33347, infoSecurePort=0, ipcPort=33657, storageInfo=lv=-57;cid=testClusterID;nsid=1138745378;c=1732422329789) 2024-11-24T04:25:31,830 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x1600f6512f5f0072 with lease ID 0xaa9cca750b8583da: from storage DS-20bb5930-23a3-441d-bdb2-015c0d56b2ec node DatanodeRegistration(127.0.0.1:34457, datanodeUuid=c4648929-9bd7-4a94-827d-36f0dcfe9f17, infoPort=33347, infoSecurePort=0, ipcPort=33657, storageInfo=lv=-57;cid=testClusterID;nsid=1138745378;c=1732422329789), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-24T04:25:31,831 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xd73db2e362fb61a8 with lease ID 0xaa9cca750b8583db: Processing first storage report for DS-11c8f594-4fc2-47e4-b222-092ec50eb991 from datanode DatanodeRegistration(127.0.0.1:36069, datanodeUuid=60946a18-8e33-40fa-966b-81f112b81c54, infoPort=41009, infoSecurePort=0, ipcPort=44559, storageInfo=lv=-57;cid=testClusterID;nsid=1138745378;c=1732422329789) 2024-11-24T04:25:31,831 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xd73db2e362fb61a8 with lease ID 0xaa9cca750b8583db: from storage DS-11c8f594-4fc2-47e4-b222-092ec50eb991 node DatanodeRegistration(127.0.0.1:36069, datanodeUuid=60946a18-8e33-40fa-966b-81f112b81c54, infoPort=41009, infoSecurePort=0, ipcPort=44559, storageInfo=lv=-57;cid=testClusterID;nsid=1138745378;c=1732422329789), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-24T04:25:31,831 WARN [Thread-111 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-24T04:25:31,831 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x1600f6512f5f0072 with lease ID 0xaa9cca750b8583da: Processing first storage report for DS-0e934c5a-2dd8-4781-b080-767081d585a2 from datanode DatanodeRegistration(127.0.0.1:34457, datanodeUuid=c4648929-9bd7-4a94-827d-36f0dcfe9f17, infoPort=33347, infoSecurePort=0, ipcPort=33657, storageInfo=lv=-57;cid=testClusterID;nsid=1138745378;c=1732422329789) 2024-11-24T04:25:31,831 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x1600f6512f5f0072 with lease ID 0xaa9cca750b8583da: from storage DS-0e934c5a-2dd8-4781-b080-767081d585a2 node DatanodeRegistration(127.0.0.1:34457, datanodeUuid=c4648929-9bd7-4a94-827d-36f0dcfe9f17, infoPort=33347, infoSecurePort=0, ipcPort=33657, storageInfo=lv=-57;cid=testClusterID;nsid=1138745378;c=1732422329789), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-24T04:25:31,832 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xd73db2e362fb61a8 with lease ID 0xaa9cca750b8583db: Processing first storage report for DS-d5ca7157-95b7-4d1f-9e44-bc8c1a47faa1 from datanode DatanodeRegistration(127.0.0.1:36069, datanodeUuid=60946a18-8e33-40fa-966b-81f112b81c54, infoPort=41009, infoSecurePort=0, ipcPort=44559, storageInfo=lv=-57;cid=testClusterID;nsid=1138745378;c=1732422329789) 2024-11-24T04:25:31,832 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xd73db2e362fb61a8 with lease ID 0xaa9cca750b8583db: from storage DS-d5ca7157-95b7-4d1f-9e44-bc8c1a47faa1 node DatanodeRegistration(127.0.0.1:36069, datanodeUuid=60946a18-8e33-40fa-966b-81f112b81c54, infoPort=41009, infoSecurePort=0, ipcPort=44559, storageInfo=lv=-57;cid=testClusterID;nsid=1138745378;c=1732422329789), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-24T04:25:31,837 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xc7c53b86ae4e2acb with lease ID 0xaa9cca750b8583dc: Processing first storage report for DS-54254ac8-0fae-43f7-ab06-53faa0e05dd7 from datanode DatanodeRegistration(127.0.0.1:44781, datanodeUuid=d070398a-b698-4d3a-9f67-a5eac0f5f265, infoPort=35691, infoSecurePort=0, ipcPort=38147, storageInfo=lv=-57;cid=testClusterID;nsid=1138745378;c=1732422329789) 2024-11-24T04:25:31,837 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xc7c53b86ae4e2acb with lease ID 0xaa9cca750b8583dc: from storage DS-54254ac8-0fae-43f7-ab06-53faa0e05dd7 node DatanodeRegistration(127.0.0.1:44781, datanodeUuid=d070398a-b698-4d3a-9f67-a5eac0f5f265, infoPort=35691, infoSecurePort=0, ipcPort=38147, storageInfo=lv=-57;cid=testClusterID;nsid=1138745378;c=1732422329789), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-24T04:25:31,838 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xc7c53b86ae4e2acb with lease ID 0xaa9cca750b8583dc: Processing first storage report for DS-9c61f371-08eb-4b66-8174-dcf0c18d1617 from datanode DatanodeRegistration(127.0.0.1:44781, datanodeUuid=d070398a-b698-4d3a-9f67-a5eac0f5f265, infoPort=35691, infoSecurePort=0, ipcPort=38147, storageInfo=lv=-57;cid=testClusterID;nsid=1138745378;c=1732422329789) 2024-11-24T04:25:31,838 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xc7c53b86ae4e2acb with lease ID 0xaa9cca750b8583dc: from storage DS-9c61f371-08eb-4b66-8174-dcf0c18d1617 node DatanodeRegistration(127.0.0.1:44781, datanodeUuid=d070398a-b698-4d3a-9f67-a5eac0f5f265, infoPort=35691, infoSecurePort=0, ipcPort=38147, storageInfo=lv=-57;cid=testClusterID;nsid=1138745378;c=1732422329789), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-24T04:25:32,027 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6816aaf7-45a5-053e-0edc-4c140671afba 2024-11-24T04:25:32,104 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6816aaf7-45a5-053e-0edc-4c140671afba/cluster_be0b9e06-9b41-befd-1079-43744c8c01cc/zookeeper_0, clientPort=62017, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6816aaf7-45a5-053e-0edc-4c140671afba/cluster_be0b9e06-9b41-befd-1079-43744c8c01cc/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6816aaf7-45a5-053e-0edc-4c140671afba/cluster_be0b9e06-9b41-befd-1079-43744c8c01cc/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-24T04:25:32,114 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=62017 2024-11-24T04:25:32,137 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-24T04:25:32,141 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-24T04:25:32,372 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36069 is added to blk_1073741825_1001 (size=7) 2024-11-24T04:25:32,372 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44781 is added to blk_1073741825_1001 (size=7) 2024-11-24T04:25:32,373 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34457 is added to blk_1073741825_1001 (size=7) 2024-11-24T04:25:32,780 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:44611/user/jenkins/test-data/33c38499-d2fe-b6f0-8d8b-5809759afd19 with version=8 2024-11-24T04:25:32,780 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:44611/user/jenkins/test-data/33c38499-d2fe-b6f0-8d8b-5809759afd19/hbase-staging 2024-11-24T04:25:33,090 INFO [Time-limited test {}] client.ConnectionUtils(128): master/fadb2fe0ba32:0 server-side Connection retries=45 2024-11-24T04:25:33,101 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-24T04:25:33,102 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-24T04:25:33,107 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-24T04:25:33,108 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-24T04:25:33,108 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-24T04:25:33,250 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-24T04:25:33,314 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-24T04:25:33,323 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-24T04:25:33,327 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-24T04:25:33,358 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 7307 (auto-detected) 2024-11-24T04:25:33,359 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-24T04:25:33,379 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:33969 2024-11-24T04:25:33,400 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:33969 connecting to ZooKeeper ensemble=127.0.0.1:62017 2024-11-24T04:25:33,436 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:339690x0, quorum=127.0.0.1:62017, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-24T04:25:33,439 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:33969-0x1014838d9450000 connected 2024-11-24T04:25:33,469 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-24T04:25:33,473 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-24T04:25:33,483 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:33969-0x1014838d9450000, quorum=127.0.0.1:62017, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-24T04:25:33,487 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:44611/user/jenkins/test-data/33c38499-d2fe-b6f0-8d8b-5809759afd19, hbase.cluster.distributed=false 2024-11-24T04:25:33,512 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:33969-0x1014838d9450000, quorum=127.0.0.1:62017, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-24T04:25:33,516 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=33969 2024-11-24T04:25:33,517 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=33969 2024-11-24T04:25:33,518 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=33969 2024-11-24T04:25:33,518 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=33969 2024-11-24T04:25:33,519 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=33969 2024-11-24T04:25:33,641 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/fadb2fe0ba32:0 server-side Connection retries=45 2024-11-24T04:25:33,643 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-24T04:25:33,644 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-24T04:25:33,644 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-24T04:25:33,644 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-24T04:25:33,644 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-24T04:25:33,647 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-24T04:25:33,650 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-24T04:25:33,651 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:35977 2024-11-24T04:25:33,654 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:35977 connecting to ZooKeeper ensemble=127.0.0.1:62017 2024-11-24T04:25:33,656 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-24T04:25:33,659 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-24T04:25:33,665 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:359770x0, quorum=127.0.0.1:62017, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-24T04:25:33,666 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:35977-0x1014838d9450001 connected 2024-11-24T04:25:33,666 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:35977-0x1014838d9450001, quorum=127.0.0.1:62017, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-24T04:25:33,670 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-24T04:25:33,678 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-24T04:25:33,680 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:35977-0x1014838d9450001, quorum=127.0.0.1:62017, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-24T04:25:33,685 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:35977-0x1014838d9450001, quorum=127.0.0.1:62017, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-24T04:25:33,686 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=35977 2024-11-24T04:25:33,686 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=35977 2024-11-24T04:25:33,686 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=35977 2024-11-24T04:25:33,687 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=35977 2024-11-24T04:25:33,688 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=35977 2024-11-24T04:25:33,706 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/fadb2fe0ba32:0 server-side Connection retries=45 2024-11-24T04:25:33,706 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-24T04:25:33,706 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-24T04:25:33,707 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-24T04:25:33,707 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-24T04:25:33,707 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-24T04:25:33,707 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-24T04:25:33,707 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-24T04:25:33,708 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:43811 2024-11-24T04:25:33,710 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:43811 connecting to ZooKeeper ensemble=127.0.0.1:62017 2024-11-24T04:25:33,711 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-24T04:25:33,714 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-24T04:25:33,725 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:438110x0, quorum=127.0.0.1:62017, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-24T04:25:33,726 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:43811-0x1014838d9450002 connected 2024-11-24T04:25:33,726 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:43811-0x1014838d9450002, quorum=127.0.0.1:62017, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-24T04:25:33,726 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-24T04:25:33,727 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-24T04:25:33,728 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:43811-0x1014838d9450002, quorum=127.0.0.1:62017, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-24T04:25:33,730 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:43811-0x1014838d9450002, quorum=127.0.0.1:62017, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-24T04:25:33,732 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=43811 2024-11-24T04:25:33,733 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=43811 2024-11-24T04:25:33,733 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=43811 2024-11-24T04:25:33,734 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=43811 2024-11-24T04:25:33,734 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=43811 2024-11-24T04:25:33,752 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/fadb2fe0ba32:0 server-side Connection retries=45 2024-11-24T04:25:33,752 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-24T04:25:33,752 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-24T04:25:33,752 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-24T04:25:33,752 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-24T04:25:33,753 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-24T04:25:33,753 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-24T04:25:33,753 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-24T04:25:33,754 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:45209 2024-11-24T04:25:33,755 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:45209 connecting to ZooKeeper ensemble=127.0.0.1:62017 2024-11-24T04:25:33,756 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-24T04:25:33,758 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-24T04:25:33,765 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:452090x0, quorum=127.0.0.1:62017, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-24T04:25:33,766 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:452090x0, quorum=127.0.0.1:62017, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-24T04:25:33,766 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:45209-0x1014838d9450003 connected 2024-11-24T04:25:33,767 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-24T04:25:33,768 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-24T04:25:33,769 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:45209-0x1014838d9450003, quorum=127.0.0.1:62017, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-24T04:25:33,771 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:45209-0x1014838d9450003, quorum=127.0.0.1:62017, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-24T04:25:33,772 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=45209 2024-11-24T04:25:33,772 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=45209 2024-11-24T04:25:33,773 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=45209 2024-11-24T04:25:33,776 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=45209 2024-11-24T04:25:33,777 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=45209 2024-11-24T04:25:33,791 DEBUG [M:0;fadb2fe0ba32:33969 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;fadb2fe0ba32:33969 2024-11-24T04:25:33,792 INFO [master/fadb2fe0ba32:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/fadb2fe0ba32,33969,1732422332893 2024-11-24T04:25:33,799 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35977-0x1014838d9450001, quorum=127.0.0.1:62017, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-24T04:25:33,799 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43811-0x1014838d9450002, quorum=127.0.0.1:62017, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-24T04:25:33,799 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33969-0x1014838d9450000, quorum=127.0.0.1:62017, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-24T04:25:33,799 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45209-0x1014838d9450003, quorum=127.0.0.1:62017, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-24T04:25:33,802 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:33969-0x1014838d9450000, quorum=127.0.0.1:62017, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/fadb2fe0ba32,33969,1732422332893 2024-11-24T04:25:33,822 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35977-0x1014838d9450001, quorum=127.0.0.1:62017, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-24T04:25:33,822 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43811-0x1014838d9450002, quorum=127.0.0.1:62017, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-24T04:25:33,822 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45209-0x1014838d9450003, quorum=127.0.0.1:62017, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-24T04:25:33,822 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33969-0x1014838d9450000, quorum=127.0.0.1:62017, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T04:25:33,823 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45209-0x1014838d9450003, quorum=127.0.0.1:62017, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T04:25:33,823 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43811-0x1014838d9450002, quorum=127.0.0.1:62017, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T04:25:33,823 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35977-0x1014838d9450001, quorum=127.0.0.1:62017, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T04:25:33,823 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:33969-0x1014838d9450000, quorum=127.0.0.1:62017, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-24T04:25:33,825 INFO [master/fadb2fe0ba32:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/fadb2fe0ba32,33969,1732422332893 from backup master directory 2024-11-24T04:25:33,828 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33969-0x1014838d9450000, quorum=127.0.0.1:62017, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/fadb2fe0ba32,33969,1732422332893 2024-11-24T04:25:33,828 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43811-0x1014838d9450002, quorum=127.0.0.1:62017, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-24T04:25:33,828 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35977-0x1014838d9450001, quorum=127.0.0.1:62017, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-24T04:25:33,828 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45209-0x1014838d9450003, quorum=127.0.0.1:62017, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-24T04:25:33,828 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33969-0x1014838d9450000, quorum=127.0.0.1:62017, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-24T04:25:33,829 WARN [master/fadb2fe0ba32:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-24T04:25:33,829 INFO [master/fadb2fe0ba32:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=fadb2fe0ba32,33969,1732422332893 2024-11-24T04:25:33,831 INFO [master/fadb2fe0ba32:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-24T04:25:33,833 INFO [master/fadb2fe0ba32:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-24T04:25:33,890 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:44611/user/jenkins/test-data/33c38499-d2fe-b6f0-8d8b-5809759afd19/hbase.id] with ID: 19ccd069-e1d9-4899-ace9-8d275717a5ba 2024-11-24T04:25:33,890 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:44611/user/jenkins/test-data/33c38499-d2fe-b6f0-8d8b-5809759afd19/.tmp/hbase.id 2024-11-24T04:25:33,903 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36069 is added to blk_1073741826_1002 (size=42) 2024-11-24T04:25:33,903 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44781 is added to blk_1073741826_1002 (size=42) 2024-11-24T04:25:33,904 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34457 is added to blk_1073741826_1002 (size=42) 2024-11-24T04:25:33,905 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:44611/user/jenkins/test-data/33c38499-d2fe-b6f0-8d8b-5809759afd19/.tmp/hbase.id]:[hdfs://localhost:44611/user/jenkins/test-data/33c38499-d2fe-b6f0-8d8b-5809759afd19/hbase.id] 2024-11-24T04:25:33,947 INFO [master/fadb2fe0ba32:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-24T04:25:33,951 INFO [master/fadb2fe0ba32:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-24T04:25:33,971 INFO [master/fadb2fe0ba32:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 18ms. 2024-11-24T04:25:33,975 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43811-0x1014838d9450002, quorum=127.0.0.1:62017, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T04:25:33,975 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33969-0x1014838d9450000, quorum=127.0.0.1:62017, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T04:25:33,975 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45209-0x1014838d9450003, quorum=127.0.0.1:62017, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T04:25:33,975 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35977-0x1014838d9450001, quorum=127.0.0.1:62017, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T04:25:33,997 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44781 is added to blk_1073741827_1003 (size=196) 2024-11-24T04:25:33,998 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34457 is added to blk_1073741827_1003 (size=196) 2024-11-24T04:25:33,998 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36069 is added to blk_1073741827_1003 (size=196) 2024-11-24T04:25:34,017 INFO [master/fadb2fe0ba32:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-24T04:25:34,019 INFO [master/fadb2fe0ba32:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-24T04:25:34,025 INFO [master/fadb2fe0ba32:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T04:25:34,059 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34457 is added to blk_1073741828_1004 (size=1189) 2024-11-24T04:25:34,059 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36069 is added to blk_1073741828_1004 (size=1189) 2024-11-24T04:25:34,060 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44781 is added to blk_1073741828_1004 (size=1189) 2024-11-24T04:25:34,079 INFO [master/fadb2fe0ba32:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:44611/user/jenkins/test-data/33c38499-d2fe-b6f0-8d8b-5809759afd19/MasterData/data/master/store 2024-11-24T04:25:34,101 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36069 is added to blk_1073741829_1005 (size=34) 2024-11-24T04:25:34,101 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34457 is added to blk_1073741829_1005 (size=34) 2024-11-24T04:25:34,102 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44781 is added to blk_1073741829_1005 (size=34) 2024-11-24T04:25:34,109 INFO [master/fadb2fe0ba32:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-24T04:25:34,113 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T04:25:34,114 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-24T04:25:34,114 INFO [master/fadb2fe0ba32:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T04:25:34,114 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T04:25:34,116 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-24T04:25:34,116 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T04:25:34,116 INFO [master/fadb2fe0ba32:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T04:25:34,117 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732422334114Disabling compacts and flushes for region at 1732422334114Disabling writes for close at 1732422334116 (+2 ms)Writing region close event to WAL at 1732422334116Closed at 1732422334116 2024-11-24T04:25:34,119 WARN [master/fadb2fe0ba32:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:44611/user/jenkins/test-data/33c38499-d2fe-b6f0-8d8b-5809759afd19/MasterData/data/master/store/.initializing 2024-11-24T04:25:34,120 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:44611/user/jenkins/test-data/33c38499-d2fe-b6f0-8d8b-5809759afd19/MasterData/WALs/fadb2fe0ba32,33969,1732422332893 2024-11-24T04:25:34,128 INFO [master/fadb2fe0ba32:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-24T04:25:34,143 INFO [master/fadb2fe0ba32:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=fadb2fe0ba32%2C33969%2C1732422332893, suffix=, logDir=hdfs://localhost:44611/user/jenkins/test-data/33c38499-d2fe-b6f0-8d8b-5809759afd19/MasterData/WALs/fadb2fe0ba32,33969,1732422332893, archiveDir=hdfs://localhost:44611/user/jenkins/test-data/33c38499-d2fe-b6f0-8d8b-5809759afd19/MasterData/oldWALs, maxLogs=10 2024-11-24T04:25:34,176 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/33c38499-d2fe-b6f0-8d8b-5809759afd19/MasterData/WALs/fadb2fe0ba32,33969,1732422332893/fadb2fe0ba32%2C33969%2C1732422332893.1732422334148, exclude list is [], retry=0 2024-11-24T04:25:34,196 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.initialize(FanOutOneBlockAsyncDFSOutputHelper.java:413) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$5.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:472) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$5.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:467) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.NettyFutureUtils.lambda$addListener$0(NettyFutureUtils.java:56) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setSuccess0(DefaultPromise.java:625) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.trySuccess(DefaultPromise.java:105) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPromise.trySuccess(DefaultChannelPromise.java:84) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:658) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:696) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T04:25:34,201 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36069,DS-11c8f594-4fc2-47e4-b222-092ec50eb991,DISK] 2024-11-24T04:25:34,201 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34457,DS-20bb5930-23a3-441d-bdb2-015c0d56b2ec,DISK] 2024-11-24T04:25:34,201 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44781,DS-54254ac8-0fae-43f7-ab06-53faa0e05dd7,DISK] 2024-11-24T04:25:34,204 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-24T04:25:34,246 INFO [master/fadb2fe0ba32:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/33c38499-d2fe-b6f0-8d8b-5809759afd19/MasterData/WALs/fadb2fe0ba32,33969,1732422332893/fadb2fe0ba32%2C33969%2C1732422332893.1732422334148 2024-11-24T04:25:34,247 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33347:33347),(127.0.0.1/127.0.0.1:35691:35691),(127.0.0.1/127.0.0.1:41009:41009)] 2024-11-24T04:25:34,248 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-24T04:25:34,248 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T04:25:34,252 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-24T04:25:34,253 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-24T04:25:34,291 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-24T04:25:34,319 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-24T04:25:34,323 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:25:34,325 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T04:25:34,326 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-24T04:25:34,329 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-24T04:25:34,329 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:25:34,330 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:25:34,331 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-24T04:25:34,333 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-24T04:25:34,333 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:25:34,334 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:25:34,334 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-24T04:25:34,337 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-24T04:25:34,337 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:25:34,338 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:25:34,338 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-24T04:25:34,342 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44611/user/jenkins/test-data/33c38499-d2fe-b6f0-8d8b-5809759afd19/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-24T04:25:34,343 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44611/user/jenkins/test-data/33c38499-d2fe-b6f0-8d8b-5809759afd19/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-24T04:25:34,348 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-24T04:25:34,349 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-24T04:25:34,352 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-24T04:25:34,355 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-24T04:25:34,360 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44611/user/jenkins/test-data/33c38499-d2fe-b6f0-8d8b-5809759afd19/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T04:25:34,361 INFO [master/fadb2fe0ba32:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62970286, jitterRate=-0.061669617891311646}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-24T04:25:34,367 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1732422334265Initializing all the Stores at 1732422334267 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732422334268 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732422334268Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732422334269 (+1 ms)Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732422334269Cleaning up temporary data from old regions at 1732422334349 (+80 ms)Region opened successfully at 1732422334367 (+18 ms) 2024-11-24T04:25:34,369 INFO [master/fadb2fe0ba32:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-24T04:25:34,406 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@33248c7a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=fadb2fe0ba32/172.17.0.2:0 2024-11-24T04:25:34,449 INFO [master/fadb2fe0ba32:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-24T04:25:34,461 INFO [master/fadb2fe0ba32:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-24T04:25:34,461 INFO [master/fadb2fe0ba32:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-24T04:25:34,464 INFO [master/fadb2fe0ba32:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-24T04:25:34,466 INFO [master/fadb2fe0ba32:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-24T04:25:34,471 INFO [master/fadb2fe0ba32:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-11-24T04:25:34,471 INFO [master/fadb2fe0ba32:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-24T04:25:34,504 INFO [master/fadb2fe0ba32:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-24T04:25:34,516 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33969-0x1014838d9450000, quorum=127.0.0.1:62017, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-24T04:25:34,518 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-24T04:25:34,521 INFO [master/fadb2fe0ba32:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-24T04:25:34,523 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33969-0x1014838d9450000, quorum=127.0.0.1:62017, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-24T04:25:34,524 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-24T04:25:34,527 INFO [master/fadb2fe0ba32:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-24T04:25:34,530 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33969-0x1014838d9450000, quorum=127.0.0.1:62017, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-24T04:25:34,533 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-24T04:25:34,534 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33969-0x1014838d9450000, quorum=127.0.0.1:62017, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-24T04:25:34,536 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-24T04:25:34,555 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33969-0x1014838d9450000, quorum=127.0.0.1:62017, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-24T04:25:34,557 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-24T04:25:34,561 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43811-0x1014838d9450002, quorum=127.0.0.1:62017, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-24T04:25:34,561 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45209-0x1014838d9450003, quorum=127.0.0.1:62017, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-24T04:25:34,561 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35977-0x1014838d9450001, quorum=127.0.0.1:62017, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-24T04:25:34,561 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33969-0x1014838d9450000, quorum=127.0.0.1:62017, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-24T04:25:34,561 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43811-0x1014838d9450002, quorum=127.0.0.1:62017, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T04:25:34,561 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35977-0x1014838d9450001, quorum=127.0.0.1:62017, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T04:25:34,561 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45209-0x1014838d9450003, quorum=127.0.0.1:62017, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T04:25:34,561 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33969-0x1014838d9450000, quorum=127.0.0.1:62017, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T04:25:34,565 INFO [master/fadb2fe0ba32:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=fadb2fe0ba32,33969,1732422332893, sessionid=0x1014838d9450000, setting cluster-up flag (Was=false) 2024-11-24T04:25:34,584 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43811-0x1014838d9450002, quorum=127.0.0.1:62017, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T04:25:34,584 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33969-0x1014838d9450000, quorum=127.0.0.1:62017, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T04:25:34,584 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45209-0x1014838d9450003, quorum=127.0.0.1:62017, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T04:25:34,584 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35977-0x1014838d9450001, quorum=127.0.0.1:62017, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T04:25:34,592 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-24T04:25:34,593 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=fadb2fe0ba32,33969,1732422332893 2024-11-24T04:25:34,598 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33969-0x1014838d9450000, quorum=127.0.0.1:62017, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T04:25:34,598 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43811-0x1014838d9450002, quorum=127.0.0.1:62017, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T04:25:34,598 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45209-0x1014838d9450003, quorum=127.0.0.1:62017, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T04:25:34,599 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35977-0x1014838d9450001, quorum=127.0.0.1:62017, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T04:25:34,606 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-24T04:25:34,607 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=fadb2fe0ba32,33969,1732422332893 2024-11-24T04:25:34,614 INFO [master/fadb2fe0ba32:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:44611/user/jenkins/test-data/33c38499-d2fe-b6f0-8d8b-5809759afd19/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-24T04:25:34,681 INFO [RS:1;fadb2fe0ba32:43811 {}] regionserver.HRegionServer(746): ClusterId : 19ccd069-e1d9-4899-ace9-8d275717a5ba 2024-11-24T04:25:34,681 INFO [RS:2;fadb2fe0ba32:45209 {}] regionserver.HRegionServer(746): ClusterId : 19ccd069-e1d9-4899-ace9-8d275717a5ba 2024-11-24T04:25:34,681 INFO [RS:0;fadb2fe0ba32:35977 {}] regionserver.HRegionServer(746): ClusterId : 19ccd069-e1d9-4899-ace9-8d275717a5ba 2024-11-24T04:25:34,683 DEBUG [RS:1;fadb2fe0ba32:43811 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-24T04:25:34,684 DEBUG [RS:0;fadb2fe0ba32:35977 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-24T04:25:34,683 DEBUG [RS:2;fadb2fe0ba32:45209 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-24T04:25:34,689 DEBUG [RS:2;fadb2fe0ba32:45209 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-24T04:25:34,689 DEBUG [RS:0;fadb2fe0ba32:35977 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-24T04:25:34,689 DEBUG [RS:2;fadb2fe0ba32:45209 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-24T04:25:34,689 DEBUG [RS:1;fadb2fe0ba32:43811 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-24T04:25:34,689 DEBUG [RS:0;fadb2fe0ba32:35977 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-24T04:25:34,689 DEBUG [RS:1;fadb2fe0ba32:43811 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-24T04:25:34,694 DEBUG [RS:2;fadb2fe0ba32:45209 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-24T04:25:34,694 DEBUG [RS:0;fadb2fe0ba32:35977 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-24T04:25:34,695 DEBUG [RS:1;fadb2fe0ba32:43811 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-24T04:25:34,695 DEBUG [RS:2;fadb2fe0ba32:45209 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d15af88, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=fadb2fe0ba32/172.17.0.2:0 2024-11-24T04:25:34,695 DEBUG [RS:1;fadb2fe0ba32:43811 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1fc2a8fa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=fadb2fe0ba32/172.17.0.2:0 2024-11-24T04:25:34,695 DEBUG [RS:0;fadb2fe0ba32:35977 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e5c9220, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=fadb2fe0ba32/172.17.0.2:0 2024-11-24T04:25:34,698 INFO [AsyncFSWAL-0-hdfs://localhost:44611/user/jenkins/test-data/33c38499-d2fe-b6f0-8d8b-5809759afd19/MasterData-prefix:fadb2fe0ba32,33969,1732422332893 {}] compress.Compression(560): Loaded codec org.apache.hadoop.hbase.io.compress.ReusableStreamGzipCodec for compression algorithm GZ 2024-11-24T04:25:34,712 DEBUG [RS:0;fadb2fe0ba32:35977 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;fadb2fe0ba32:35977 2024-11-24T04:25:34,713 DEBUG [RS:2;fadb2fe0ba32:45209 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:2;fadb2fe0ba32:45209 2024-11-24T04:25:34,716 INFO [RS:2;fadb2fe0ba32:45209 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-24T04:25:34,717 INFO [RS:0;fadb2fe0ba32:35977 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-24T04:25:34,717 INFO [RS:2;fadb2fe0ba32:45209 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-24T04:25:34,717 INFO [RS:0;fadb2fe0ba32:35977 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-24T04:25:34,717 DEBUG [RS:2;fadb2fe0ba32:45209 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-24T04:25:34,717 DEBUG [RS:0;fadb2fe0ba32:35977 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-24T04:25:34,719 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-24T04:25:34,720 DEBUG [RS:1;fadb2fe0ba32:43811 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;fadb2fe0ba32:43811 2024-11-24T04:25:34,720 INFO [RS:0;fadb2fe0ba32:35977 {}] regionserver.HRegionServer(2659): reportForDuty to master=fadb2fe0ba32,33969,1732422332893 with port=35977, startcode=1732422333601 2024-11-24T04:25:34,720 INFO [RS:2;fadb2fe0ba32:45209 {}] regionserver.HRegionServer(2659): reportForDuty to master=fadb2fe0ba32,33969,1732422332893 with port=45209, startcode=1732422333751 2024-11-24T04:25:34,720 INFO [RS:1;fadb2fe0ba32:43811 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-24T04:25:34,721 INFO [RS:1;fadb2fe0ba32:43811 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-24T04:25:34,721 DEBUG [RS:1;fadb2fe0ba32:43811 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-24T04:25:34,722 INFO [RS:1;fadb2fe0ba32:43811 {}] regionserver.HRegionServer(2659): reportForDuty to master=fadb2fe0ba32,33969,1732422332893 with port=43811, startcode=1732422333705 2024-11-24T04:25:34,730 INFO [master/fadb2fe0ba32:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-24T04:25:34,734 DEBUG [RS:1;fadb2fe0ba32:43811 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-24T04:25:34,734 DEBUG [RS:2;fadb2fe0ba32:45209 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-24T04:25:34,734 DEBUG [RS:0;fadb2fe0ba32:35977 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-24T04:25:34,737 INFO [master/fadb2fe0ba32:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-24T04:25:34,744 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: fadb2fe0ba32,33969,1732422332893 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-24T04:25:34,752 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/fadb2fe0ba32:0, corePoolSize=5, maxPoolSize=5 2024-11-24T04:25:34,752 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/fadb2fe0ba32:0, corePoolSize=5, maxPoolSize=5 2024-11-24T04:25:34,752 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/fadb2fe0ba32:0, corePoolSize=5, maxPoolSize=5 2024-11-24T04:25:34,752 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/fadb2fe0ba32:0, corePoolSize=5, maxPoolSize=5 2024-11-24T04:25:34,752 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/fadb2fe0ba32:0, corePoolSize=10, maxPoolSize=10 2024-11-24T04:25:34,752 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/fadb2fe0ba32:0, corePoolSize=1, maxPoolSize=1 2024-11-24T04:25:34,753 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/fadb2fe0ba32:0, corePoolSize=2, maxPoolSize=2 2024-11-24T04:25:34,753 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/fadb2fe0ba32:0, corePoolSize=1, maxPoolSize=1 2024-11-24T04:25:34,761 INFO [master/fadb2fe0ba32:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732422364761 2024-11-24T04:25:34,763 INFO [master/fadb2fe0ba32:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-24T04:25:34,764 INFO [master/fadb2fe0ba32:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-24T04:25:34,769 INFO [master/fadb2fe0ba32:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-24T04:25:34,769 INFO [master/fadb2fe0ba32:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-24T04:25:34,769 INFO [master/fadb2fe0ba32:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-24T04:25:34,770 INFO [master/fadb2fe0ba32:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-24T04:25:34,771 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-24T04:25:34,771 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-24T04:25:34,772 INFO [master/fadb2fe0ba32:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-24T04:25:34,776 INFO [master/fadb2fe0ba32:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-24T04:25:34,777 INFO [master/fadb2fe0ba32:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-24T04:25:34,778 INFO [HMaster-EventLoopGroup-2-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42193, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-24T04:25:34,778 INFO [HMaster-EventLoopGroup-2-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36631, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.2 (auth:SIMPLE), service=RegionServerStatusService 2024-11-24T04:25:34,778 INFO [HMaster-EventLoopGroup-2-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44583, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.1 (auth:SIMPLE), service=RegionServerStatusService 2024-11-24T04:25:34,778 INFO [master/fadb2fe0ba32:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-24T04:25:34,780 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:25:34,781 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-24T04:25:34,782 INFO [master/fadb2fe0ba32:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-24T04:25:34,782 INFO [master/fadb2fe0ba32:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-24T04:25:34,784 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/fadb2fe0ba32:0:becomeActiveMaster-HFileCleaner.large.0-1732422334783,5,FailOnTimeoutGroup] 2024-11-24T04:25:34,785 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33969 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T04:25:34,790 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33969 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T04:25:34,791 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/fadb2fe0ba32:0:becomeActiveMaster-HFileCleaner.small.0-1732422334784,5,FailOnTimeoutGroup] 2024-11-24T04:25:34,791 INFO [master/fadb2fe0ba32:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-24T04:25:34,791 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33969 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T04:25:34,792 INFO [master/fadb2fe0ba32:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-24T04:25:34,793 INFO [master/fadb2fe0ba32:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-24T04:25:34,793 INFO [master/fadb2fe0ba32:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-24T04:25:34,850 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36069 is added to blk_1073741831_1007 (size=1321) 2024-11-24T04:25:34,851 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34457 is added to blk_1073741831_1007 (size=1321) 2024-11-24T04:25:34,851 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44781 is added to blk_1073741831_1007 (size=1321) 2024-11-24T04:25:34,854 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:44611/user/jenkins/test-data/33c38499-d2fe-b6f0-8d8b-5809759afd19/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-24T04:25:34,854 DEBUG [RS:0;fadb2fe0ba32:35977 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-11-24T04:25:34,854 DEBUG [RS:1;fadb2fe0ba32:43811 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-11-24T04:25:34,854 DEBUG [RS:2;fadb2fe0ba32:45209 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-11-24T04:25:34,854 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:44611/user/jenkins/test-data/33c38499-d2fe-b6f0-8d8b-5809759afd19 2024-11-24T04:25:34,854 WARN [RS:1;fadb2fe0ba32:43811 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-11-24T04:25:34,854 WARN [RS:0;fadb2fe0ba32:35977 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-11-24T04:25:34,854 WARN [RS:2;fadb2fe0ba32:45209 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-11-24T04:25:34,866 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36069 is added to blk_1073741832_1008 (size=32) 2024-11-24T04:25:34,867 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44781 is added to blk_1073741832_1008 (size=32) 2024-11-24T04:25:34,867 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34457 is added to blk_1073741832_1008 (size=32) 2024-11-24T04:25:34,869 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T04:25:34,872 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-24T04:25:34,874 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-24T04:25:34,874 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:25:34,875 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T04:25:34,876 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-24T04:25:34,878 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-24T04:25:34,878 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:25:34,879 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T04:25:34,879 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-24T04:25:34,881 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-24T04:25:34,881 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:25:34,882 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T04:25:34,882 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-24T04:25:34,884 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-24T04:25:34,884 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:25:34,885 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T04:25:34,885 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-24T04:25:34,886 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44611/user/jenkins/test-data/33c38499-d2fe-b6f0-8d8b-5809759afd19/data/hbase/meta/1588230740 2024-11-24T04:25:34,887 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44611/user/jenkins/test-data/33c38499-d2fe-b6f0-8d8b-5809759afd19/data/hbase/meta/1588230740 2024-11-24T04:25:34,889 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-24T04:25:34,889 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-24T04:25:34,890 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-24T04:25:34,893 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-24T04:25:34,897 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44611/user/jenkins/test-data/33c38499-d2fe-b6f0-8d8b-5809759afd19/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T04:25:34,898 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73745606, jitterRate=0.09889516234397888}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-24T04:25:34,900 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1732422334869Initializing all the Stores at 1732422334871 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732422334871Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732422334871Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732422334872 (+1 ms)Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732422334872Cleaning up temporary data from old regions at 1732422334889 (+17 ms)Region opened successfully at 1732422334900 (+11 ms) 2024-11-24T04:25:34,901 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-24T04:25:34,901 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-24T04:25:34,901 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-24T04:25:34,901 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-24T04:25:34,901 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-24T04:25:34,902 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-24T04:25:34,902 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732422334901Disabling compacts and flushes for region at 1732422334901Disabling writes for close at 1732422334901Writing region close event to WAL at 1732422334902 (+1 ms)Closed at 1732422334902 2024-11-24T04:25:34,906 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-24T04:25:34,906 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-24T04:25:34,913 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-24T04:25:34,922 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-24T04:25:34,926 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-24T04:25:34,956 INFO [RS:1;fadb2fe0ba32:43811 {}] regionserver.HRegionServer(2659): reportForDuty to master=fadb2fe0ba32,33969,1732422332893 with port=43811, startcode=1732422333705 2024-11-24T04:25:34,956 INFO [RS:0;fadb2fe0ba32:35977 {}] regionserver.HRegionServer(2659): reportForDuty to master=fadb2fe0ba32,33969,1732422332893 with port=35977, startcode=1732422333601 2024-11-24T04:25:34,956 INFO [RS:2;fadb2fe0ba32:45209 {}] regionserver.HRegionServer(2659): reportForDuty to master=fadb2fe0ba32,33969,1732422332893 with port=45209, startcode=1732422333751 2024-11-24T04:25:34,958 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33969 {}] master.ServerManager(363): Checking decommissioned status of RegionServer fadb2fe0ba32,35977,1732422333601 2024-11-24T04:25:34,961 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33969 {}] master.ServerManager(517): Registering regionserver=fadb2fe0ba32,35977,1732422333601 2024-11-24T04:25:34,970 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33969 {}] master.ServerManager(363): Checking decommissioned status of RegionServer fadb2fe0ba32,45209,1732422333751 2024-11-24T04:25:34,970 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33969 {}] master.ServerManager(517): Registering regionserver=fadb2fe0ba32,45209,1732422333751 2024-11-24T04:25:34,970 DEBUG [RS:0;fadb2fe0ba32:35977 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:44611/user/jenkins/test-data/33c38499-d2fe-b6f0-8d8b-5809759afd19 2024-11-24T04:25:34,970 DEBUG [RS:0;fadb2fe0ba32:35977 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:44611 2024-11-24T04:25:34,970 DEBUG [RS:0;fadb2fe0ba32:35977 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-24T04:25:34,976 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33969 {}] master.ServerManager(363): Checking decommissioned status of RegionServer fadb2fe0ba32,43811,1732422333705 2024-11-24T04:25:34,976 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33969-0x1014838d9450000, quorum=127.0.0.1:62017, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-24T04:25:34,976 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33969 {}] master.ServerManager(517): Registering regionserver=fadb2fe0ba32,43811,1732422333705 2024-11-24T04:25:34,976 DEBUG [RS:2;fadb2fe0ba32:45209 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:44611/user/jenkins/test-data/33c38499-d2fe-b6f0-8d8b-5809759afd19 2024-11-24T04:25:34,976 DEBUG [RS:2;fadb2fe0ba32:45209 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:44611 2024-11-24T04:25:34,977 DEBUG [RS:2;fadb2fe0ba32:45209 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-24T04:25:34,978 DEBUG [RS:0;fadb2fe0ba32:35977 {}] zookeeper.ZKUtil(111): regionserver:35977-0x1014838d9450001, quorum=127.0.0.1:62017, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/fadb2fe0ba32,35977,1732422333601 2024-11-24T04:25:34,978 WARN [RS:0;fadb2fe0ba32:35977 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-24T04:25:34,978 INFO [RS:0;fadb2fe0ba32:35977 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T04:25:34,979 DEBUG [RS:0;fadb2fe0ba32:35977 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:44611/user/jenkins/test-data/33c38499-d2fe-b6f0-8d8b-5809759afd19/WALs/fadb2fe0ba32,35977,1732422333601 2024-11-24T04:25:34,985 DEBUG [RS:2;fadb2fe0ba32:45209 {}] zookeeper.ZKUtil(111): regionserver:45209-0x1014838d9450003, quorum=127.0.0.1:62017, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/fadb2fe0ba32,45209,1732422333751 2024-11-24T04:25:34,985 WARN [RS:2;fadb2fe0ba32:45209 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-24T04:25:34,985 INFO [RS:2;fadb2fe0ba32:45209 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T04:25:34,985 DEBUG [RS:2;fadb2fe0ba32:45209 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:44611/user/jenkins/test-data/33c38499-d2fe-b6f0-8d8b-5809759afd19/WALs/fadb2fe0ba32,45209,1732422333751 2024-11-24T04:25:34,986 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [fadb2fe0ba32,35977,1732422333601] 2024-11-24T04:25:34,986 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [fadb2fe0ba32,45209,1732422333751] 2024-11-24T04:25:34,988 DEBUG [RS:1;fadb2fe0ba32:43811 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:44611/user/jenkins/test-data/33c38499-d2fe-b6f0-8d8b-5809759afd19 2024-11-24T04:25:34,988 DEBUG [RS:1;fadb2fe0ba32:43811 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:44611 2024-11-24T04:25:34,988 DEBUG [RS:1;fadb2fe0ba32:43811 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-24T04:25:34,990 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33969-0x1014838d9450000, quorum=127.0.0.1:62017, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-24T04:25:34,991 DEBUG [RS:1;fadb2fe0ba32:43811 {}] zookeeper.ZKUtil(111): regionserver:43811-0x1014838d9450002, quorum=127.0.0.1:62017, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/fadb2fe0ba32,43811,1732422333705 2024-11-24T04:25:34,991 WARN [RS:1;fadb2fe0ba32:43811 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-24T04:25:34,992 INFO [RS:1;fadb2fe0ba32:43811 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T04:25:34,992 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [fadb2fe0ba32,43811,1732422333705] 2024-11-24T04:25:34,992 DEBUG [RS:1;fadb2fe0ba32:43811 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:44611/user/jenkins/test-data/33c38499-d2fe-b6f0-8d8b-5809759afd19/WALs/fadb2fe0ba32,43811,1732422333705 2024-11-24T04:25:35,020 INFO [RS:1;fadb2fe0ba32:43811 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-24T04:25:35,020 INFO [RS:2;fadb2fe0ba32:45209 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-24T04:25:35,020 INFO [RS:0;fadb2fe0ba32:35977 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-24T04:25:35,033 INFO [RS:1;fadb2fe0ba32:43811 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-24T04:25:35,033 INFO [RS:0;fadb2fe0ba32:35977 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-24T04:25:35,034 INFO [RS:2;fadb2fe0ba32:45209 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-24T04:25:35,040 INFO [RS:0;fadb2fe0ba32:35977 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-24T04:25:35,040 INFO [RS:2;fadb2fe0ba32:45209 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-24T04:25:35,040 INFO [RS:0;fadb2fe0ba32:35977 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T04:25:35,040 INFO [RS:1;fadb2fe0ba32:43811 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-24T04:25:35,040 INFO [RS:2;fadb2fe0ba32:45209 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T04:25:35,040 INFO [RS:1;fadb2fe0ba32:43811 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T04:25:35,041 INFO [RS:1;fadb2fe0ba32:43811 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-24T04:25:35,041 INFO [RS:0;fadb2fe0ba32:35977 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-24T04:25:35,041 INFO [RS:2;fadb2fe0ba32:45209 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-24T04:25:35,047 INFO [RS:0;fadb2fe0ba32:35977 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-24T04:25:35,047 INFO [RS:2;fadb2fe0ba32:45209 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-24T04:25:35,047 INFO [RS:1;fadb2fe0ba32:43811 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-24T04:25:35,048 INFO [RS:1;fadb2fe0ba32:43811 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-24T04:25:35,048 INFO [RS:0;fadb2fe0ba32:35977 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-24T04:25:35,048 INFO [RS:2;fadb2fe0ba32:45209 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-24T04:25:35,049 DEBUG [RS:0;fadb2fe0ba32:35977 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/fadb2fe0ba32:0, corePoolSize=1, maxPoolSize=1 2024-11-24T04:25:35,049 DEBUG [RS:1;fadb2fe0ba32:43811 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/fadb2fe0ba32:0, corePoolSize=1, maxPoolSize=1 2024-11-24T04:25:35,049 DEBUG [RS:2;fadb2fe0ba32:45209 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/fadb2fe0ba32:0, corePoolSize=1, maxPoolSize=1 2024-11-24T04:25:35,049 DEBUG [RS:0;fadb2fe0ba32:35977 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/fadb2fe0ba32:0, corePoolSize=1, maxPoolSize=1 2024-11-24T04:25:35,049 DEBUG [RS:1;fadb2fe0ba32:43811 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/fadb2fe0ba32:0, corePoolSize=1, maxPoolSize=1 2024-11-24T04:25:35,049 DEBUG [RS:0;fadb2fe0ba32:35977 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/fadb2fe0ba32:0, corePoolSize=1, maxPoolSize=1 2024-11-24T04:25:35,049 DEBUG [RS:2;fadb2fe0ba32:45209 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/fadb2fe0ba32:0, corePoolSize=1, maxPoolSize=1 2024-11-24T04:25:35,049 DEBUG [RS:1;fadb2fe0ba32:43811 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/fadb2fe0ba32:0, corePoolSize=1, maxPoolSize=1 2024-11-24T04:25:35,049 DEBUG [RS:0;fadb2fe0ba32:35977 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/fadb2fe0ba32:0, corePoolSize=1, maxPoolSize=1 2024-11-24T04:25:35,049 DEBUG [RS:2;fadb2fe0ba32:45209 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/fadb2fe0ba32:0, corePoolSize=1, maxPoolSize=1 2024-11-24T04:25:35,049 DEBUG [RS:0;fadb2fe0ba32:35977 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/fadb2fe0ba32:0, corePoolSize=1, maxPoolSize=1 2024-11-24T04:25:35,049 DEBUG [RS:1;fadb2fe0ba32:43811 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/fadb2fe0ba32:0, corePoolSize=1, maxPoolSize=1 2024-11-24T04:25:35,049 DEBUG [RS:2;fadb2fe0ba32:45209 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/fadb2fe0ba32:0, corePoolSize=1, maxPoolSize=1 2024-11-24T04:25:35,050 DEBUG [RS:0;fadb2fe0ba32:35977 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/fadb2fe0ba32:0, corePoolSize=2, maxPoolSize=2 2024-11-24T04:25:35,050 DEBUG [RS:1;fadb2fe0ba32:43811 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/fadb2fe0ba32:0, corePoolSize=1, maxPoolSize=1 2024-11-24T04:25:35,050 DEBUG [RS:2;fadb2fe0ba32:45209 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/fadb2fe0ba32:0, corePoolSize=1, maxPoolSize=1 2024-11-24T04:25:35,050 DEBUG [RS:0;fadb2fe0ba32:35977 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/fadb2fe0ba32:0, corePoolSize=1, maxPoolSize=1 2024-11-24T04:25:35,050 DEBUG [RS:1;fadb2fe0ba32:43811 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/fadb2fe0ba32:0, corePoolSize=2, maxPoolSize=2 2024-11-24T04:25:35,050 DEBUG [RS:2;fadb2fe0ba32:45209 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/fadb2fe0ba32:0, corePoolSize=2, maxPoolSize=2 2024-11-24T04:25:35,050 DEBUG [RS:0;fadb2fe0ba32:35977 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/fadb2fe0ba32:0, corePoolSize=1, maxPoolSize=1 2024-11-24T04:25:35,050 DEBUG [RS:1;fadb2fe0ba32:43811 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/fadb2fe0ba32:0, corePoolSize=1, maxPoolSize=1 2024-11-24T04:25:35,050 DEBUG [RS:0;fadb2fe0ba32:35977 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/fadb2fe0ba32:0, corePoolSize=1, maxPoolSize=1 2024-11-24T04:25:35,050 DEBUG [RS:2;fadb2fe0ba32:45209 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/fadb2fe0ba32:0, corePoolSize=1, maxPoolSize=1 2024-11-24T04:25:35,050 DEBUG [RS:0;fadb2fe0ba32:35977 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/fadb2fe0ba32:0, corePoolSize=1, maxPoolSize=1 2024-11-24T04:25:35,050 DEBUG [RS:1;fadb2fe0ba32:43811 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/fadb2fe0ba32:0, corePoolSize=1, maxPoolSize=1 2024-11-24T04:25:35,050 DEBUG [RS:2;fadb2fe0ba32:45209 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/fadb2fe0ba32:0, corePoolSize=1, maxPoolSize=1 2024-11-24T04:25:35,050 DEBUG [RS:0;fadb2fe0ba32:35977 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/fadb2fe0ba32:0, corePoolSize=1, maxPoolSize=1 2024-11-24T04:25:35,050 DEBUG [RS:1;fadb2fe0ba32:43811 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/fadb2fe0ba32:0, corePoolSize=1, maxPoolSize=1 2024-11-24T04:25:35,050 DEBUG [RS:0;fadb2fe0ba32:35977 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/fadb2fe0ba32:0, corePoolSize=1, maxPoolSize=1 2024-11-24T04:25:35,050 DEBUG [RS:2;fadb2fe0ba32:45209 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/fadb2fe0ba32:0, corePoolSize=1, maxPoolSize=1 2024-11-24T04:25:35,050 DEBUG [RS:1;fadb2fe0ba32:43811 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/fadb2fe0ba32:0, corePoolSize=1, maxPoolSize=1 2024-11-24T04:25:35,050 DEBUG [RS:0;fadb2fe0ba32:35977 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/fadb2fe0ba32:0, corePoolSize=3, maxPoolSize=3 2024-11-24T04:25:35,051 DEBUG [RS:2;fadb2fe0ba32:45209 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/fadb2fe0ba32:0, corePoolSize=1, maxPoolSize=1 2024-11-24T04:25:35,051 DEBUG [RS:1;fadb2fe0ba32:43811 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/fadb2fe0ba32:0, corePoolSize=1, maxPoolSize=1 2024-11-24T04:25:35,051 DEBUG [RS:0;fadb2fe0ba32:35977 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/fadb2fe0ba32:0, corePoolSize=3, maxPoolSize=3 2024-11-24T04:25:35,051 DEBUG [RS:2;fadb2fe0ba32:45209 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/fadb2fe0ba32:0, corePoolSize=1, maxPoolSize=1 2024-11-24T04:25:35,051 DEBUG [RS:1;fadb2fe0ba32:43811 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/fadb2fe0ba32:0, corePoolSize=1, maxPoolSize=1 2024-11-24T04:25:35,051 DEBUG [RS:1;fadb2fe0ba32:43811 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/fadb2fe0ba32:0, corePoolSize=3, maxPoolSize=3 2024-11-24T04:25:35,051 DEBUG [RS:2;fadb2fe0ba32:45209 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/fadb2fe0ba32:0, corePoolSize=1, maxPoolSize=1 2024-11-24T04:25:35,051 DEBUG [RS:1;fadb2fe0ba32:43811 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/fadb2fe0ba32:0, corePoolSize=3, maxPoolSize=3 2024-11-24T04:25:35,051 DEBUG [RS:2;fadb2fe0ba32:45209 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/fadb2fe0ba32:0, corePoolSize=3, maxPoolSize=3 2024-11-24T04:25:35,051 DEBUG [RS:2;fadb2fe0ba32:45209 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/fadb2fe0ba32:0, corePoolSize=3, maxPoolSize=3 2024-11-24T04:25:35,053 INFO [RS:0;fadb2fe0ba32:35977 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-24T04:25:35,053 INFO [RS:0;fadb2fe0ba32:35977 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-24T04:25:35,053 INFO [RS:0;fadb2fe0ba32:35977 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T04:25:35,053 INFO [RS:1;fadb2fe0ba32:43811 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-24T04:25:35,053 INFO [RS:0;fadb2fe0ba32:35977 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-24T04:25:35,053 INFO [RS:2;fadb2fe0ba32:45209 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-24T04:25:35,053 INFO [RS:1;fadb2fe0ba32:43811 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-24T04:25:35,053 INFO [RS:0;fadb2fe0ba32:35977 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-24T04:25:35,053 INFO [RS:2;fadb2fe0ba32:45209 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-24T04:25:35,053 INFO [RS:1;fadb2fe0ba32:43811 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T04:25:35,053 INFO [RS:0;fadb2fe0ba32:35977 {}] hbase.ChoreService(168): Chore ScheduledChore name=fadb2fe0ba32,35977,1732422333601-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-24T04:25:35,053 INFO [RS:2;fadb2fe0ba32:45209 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T04:25:35,053 INFO [RS:1;fadb2fe0ba32:43811 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-24T04:25:35,053 INFO [RS:2;fadb2fe0ba32:45209 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-24T04:25:35,053 INFO [RS:1;fadb2fe0ba32:43811 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-24T04:25:35,054 INFO [RS:2;fadb2fe0ba32:45209 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-24T04:25:35,054 INFO [RS:1;fadb2fe0ba32:43811 {}] hbase.ChoreService(168): Chore ScheduledChore name=fadb2fe0ba32,43811,1732422333705-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-24T04:25:35,054 INFO [RS:2;fadb2fe0ba32:45209 {}] hbase.ChoreService(168): Chore ScheduledChore name=fadb2fe0ba32,45209,1732422333751-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-24T04:25:35,074 INFO [RS:0;fadb2fe0ba32:35977 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-24T04:25:35,075 INFO [RS:2;fadb2fe0ba32:45209 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-24T04:25:35,076 INFO [RS:2;fadb2fe0ba32:45209 {}] hbase.ChoreService(168): Chore ScheduledChore name=fadb2fe0ba32,45209,1732422333751-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T04:25:35,076 INFO [RS:0;fadb2fe0ba32:35977 {}] hbase.ChoreService(168): Chore ScheduledChore name=fadb2fe0ba32,35977,1732422333601-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T04:25:35,077 INFO [RS:0;fadb2fe0ba32:35977 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T04:25:35,077 INFO [RS:2;fadb2fe0ba32:45209 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T04:25:35,077 INFO [RS:0;fadb2fe0ba32:35977 {}] regionserver.Replication(171): fadb2fe0ba32,35977,1732422333601 started 2024-11-24T04:25:35,077 INFO [RS:2;fadb2fe0ba32:45209 {}] regionserver.Replication(171): fadb2fe0ba32,45209,1732422333751 started 2024-11-24T04:25:35,078 WARN [fadb2fe0ba32:33969 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-24T04:25:35,080 INFO [RS:1;fadb2fe0ba32:43811 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-24T04:25:35,080 INFO [RS:1;fadb2fe0ba32:43811 {}] hbase.ChoreService(168): Chore ScheduledChore name=fadb2fe0ba32,43811,1732422333705-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T04:25:35,080 INFO [RS:1;fadb2fe0ba32:43811 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T04:25:35,080 INFO [RS:1;fadb2fe0ba32:43811 {}] regionserver.Replication(171): fadb2fe0ba32,43811,1732422333705 started 2024-11-24T04:25:35,096 INFO [RS:0;fadb2fe0ba32:35977 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T04:25:35,096 INFO [RS:2;fadb2fe0ba32:45209 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T04:25:35,096 INFO [RS:0;fadb2fe0ba32:35977 {}] regionserver.HRegionServer(1482): Serving as fadb2fe0ba32,35977,1732422333601, RpcServer on fadb2fe0ba32/172.17.0.2:35977, sessionid=0x1014838d9450001 2024-11-24T04:25:35,096 INFO [RS:2;fadb2fe0ba32:45209 {}] regionserver.HRegionServer(1482): Serving as fadb2fe0ba32,45209,1732422333751, RpcServer on fadb2fe0ba32/172.17.0.2:45209, sessionid=0x1014838d9450003 2024-11-24T04:25:35,097 DEBUG [RS:0;fadb2fe0ba32:35977 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-24T04:25:35,097 DEBUG [RS:2;fadb2fe0ba32:45209 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-24T04:25:35,097 DEBUG [RS:0;fadb2fe0ba32:35977 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager fadb2fe0ba32,35977,1732422333601 2024-11-24T04:25:35,097 DEBUG [RS:2;fadb2fe0ba32:45209 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager fadb2fe0ba32,45209,1732422333751 2024-11-24T04:25:35,097 DEBUG [RS:0;fadb2fe0ba32:35977 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'fadb2fe0ba32,35977,1732422333601' 2024-11-24T04:25:35,097 DEBUG [RS:2;fadb2fe0ba32:45209 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'fadb2fe0ba32,45209,1732422333751' 2024-11-24T04:25:35,097 DEBUG [RS:0;fadb2fe0ba32:35977 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-24T04:25:35,097 DEBUG [RS:2;fadb2fe0ba32:45209 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-24T04:25:35,098 DEBUG [RS:0;fadb2fe0ba32:35977 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-24T04:25:35,098 DEBUG [RS:2;fadb2fe0ba32:45209 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-24T04:25:35,099 DEBUG [RS:0;fadb2fe0ba32:35977 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-24T04:25:35,099 DEBUG [RS:0;fadb2fe0ba32:35977 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-24T04:25:35,099 DEBUG [RS:0;fadb2fe0ba32:35977 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager fadb2fe0ba32,35977,1732422333601 2024-11-24T04:25:35,099 DEBUG [RS:2;fadb2fe0ba32:45209 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-24T04:25:35,099 DEBUG [RS:0;fadb2fe0ba32:35977 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'fadb2fe0ba32,35977,1732422333601' 2024-11-24T04:25:35,099 DEBUG [RS:2;fadb2fe0ba32:45209 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-24T04:25:35,099 DEBUG [RS:0;fadb2fe0ba32:35977 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-24T04:25:35,099 DEBUG [RS:2;fadb2fe0ba32:45209 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager fadb2fe0ba32,45209,1732422333751 2024-11-24T04:25:35,099 DEBUG [RS:2;fadb2fe0ba32:45209 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'fadb2fe0ba32,45209,1732422333751' 2024-11-24T04:25:35,099 DEBUG [RS:2;fadb2fe0ba32:45209 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-24T04:25:35,100 DEBUG [RS:0;fadb2fe0ba32:35977 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-24T04:25:35,100 DEBUG [RS:2;fadb2fe0ba32:45209 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-24T04:25:35,100 DEBUG [RS:0;fadb2fe0ba32:35977 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-24T04:25:35,101 INFO [RS:0;fadb2fe0ba32:35977 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-24T04:25:35,101 INFO [RS:0;fadb2fe0ba32:35977 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-24T04:25:35,101 DEBUG [RS:2;fadb2fe0ba32:45209 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-24T04:25:35,101 INFO [RS:2;fadb2fe0ba32:45209 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-24T04:25:35,101 INFO [RS:2;fadb2fe0ba32:45209 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-24T04:25:35,104 INFO [RS:1;fadb2fe0ba32:43811 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T04:25:35,104 INFO [RS:1;fadb2fe0ba32:43811 {}] regionserver.HRegionServer(1482): Serving as fadb2fe0ba32,43811,1732422333705, RpcServer on fadb2fe0ba32/172.17.0.2:43811, sessionid=0x1014838d9450002 2024-11-24T04:25:35,104 DEBUG [RS:1;fadb2fe0ba32:43811 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-24T04:25:35,104 DEBUG [RS:1;fadb2fe0ba32:43811 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager fadb2fe0ba32,43811,1732422333705 2024-11-24T04:25:35,105 DEBUG [RS:1;fadb2fe0ba32:43811 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'fadb2fe0ba32,43811,1732422333705' 2024-11-24T04:25:35,105 DEBUG [RS:1;fadb2fe0ba32:43811 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-24T04:25:35,105 DEBUG [RS:1;fadb2fe0ba32:43811 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-24T04:25:35,106 DEBUG [RS:1;fadb2fe0ba32:43811 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-24T04:25:35,106 DEBUG [RS:1;fadb2fe0ba32:43811 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-24T04:25:35,106 DEBUG [RS:1;fadb2fe0ba32:43811 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager fadb2fe0ba32,43811,1732422333705 2024-11-24T04:25:35,106 DEBUG [RS:1;fadb2fe0ba32:43811 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'fadb2fe0ba32,43811,1732422333705' 2024-11-24T04:25:35,106 DEBUG [RS:1;fadb2fe0ba32:43811 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-24T04:25:35,107 DEBUG [RS:1;fadb2fe0ba32:43811 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-24T04:25:35,107 DEBUG [RS:1;fadb2fe0ba32:43811 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-24T04:25:35,107 INFO [RS:1;fadb2fe0ba32:43811 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-24T04:25:35,107 INFO [RS:1;fadb2fe0ba32:43811 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-24T04:25:35,207 INFO [RS:0;fadb2fe0ba32:35977 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-24T04:25:35,207 INFO [RS:2;fadb2fe0ba32:45209 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-24T04:25:35,209 INFO [RS:1;fadb2fe0ba32:43811 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-24T04:25:35,211 INFO [RS:2;fadb2fe0ba32:45209 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=fadb2fe0ba32%2C45209%2C1732422333751, suffix=, logDir=hdfs://localhost:44611/user/jenkins/test-data/33c38499-d2fe-b6f0-8d8b-5809759afd19/WALs/fadb2fe0ba32,45209,1732422333751, archiveDir=hdfs://localhost:44611/user/jenkins/test-data/33c38499-d2fe-b6f0-8d8b-5809759afd19/oldWALs, maxLogs=32 2024-11-24T04:25:35,211 INFO [RS:0;fadb2fe0ba32:35977 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=fadb2fe0ba32%2C35977%2C1732422333601, suffix=, logDir=hdfs://localhost:44611/user/jenkins/test-data/33c38499-d2fe-b6f0-8d8b-5809759afd19/WALs/fadb2fe0ba32,35977,1732422333601, archiveDir=hdfs://localhost:44611/user/jenkins/test-data/33c38499-d2fe-b6f0-8d8b-5809759afd19/oldWALs, maxLogs=32 2024-11-24T04:25:35,213 INFO [RS:1;fadb2fe0ba32:43811 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=fadb2fe0ba32%2C43811%2C1732422333705, suffix=, logDir=hdfs://localhost:44611/user/jenkins/test-data/33c38499-d2fe-b6f0-8d8b-5809759afd19/WALs/fadb2fe0ba32,43811,1732422333705, archiveDir=hdfs://localhost:44611/user/jenkins/test-data/33c38499-d2fe-b6f0-8d8b-5809759afd19/oldWALs, maxLogs=32 2024-11-24T04:25:35,242 DEBUG [RS:2;fadb2fe0ba32:45209 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/33c38499-d2fe-b6f0-8d8b-5809759afd19/WALs/fadb2fe0ba32,45209,1732422333751/fadb2fe0ba32%2C45209%2C1732422333751.1732422335218, exclude list is [], retry=0 2024-11-24T04:25:35,246 DEBUG [RS:0;fadb2fe0ba32:35977 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/33c38499-d2fe-b6f0-8d8b-5809759afd19/WALs/fadb2fe0ba32,35977,1732422333601/fadb2fe0ba32%2C35977%2C1732422333601.1732422335220, exclude list is [], retry=0 2024-11-24T04:25:35,253 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36069,DS-11c8f594-4fc2-47e4-b222-092ec50eb991,DISK] 2024-11-24T04:25:35,255 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44781,DS-54254ac8-0fae-43f7-ab06-53faa0e05dd7,DISK] 2024-11-24T04:25:35,255 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34457,DS-20bb5930-23a3-441d-bdb2-015c0d56b2ec,DISK] 2024-11-24T04:25:35,256 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36069,DS-11c8f594-4fc2-47e4-b222-092ec50eb991,DISK] 2024-11-24T04:25:35,256 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44781,DS-54254ac8-0fae-43f7-ab06-53faa0e05dd7,DISK] 2024-11-24T04:25:35,257 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34457,DS-20bb5930-23a3-441d-bdb2-015c0d56b2ec,DISK] 2024-11-24T04:25:35,261 DEBUG [RS:1;fadb2fe0ba32:43811 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/33c38499-d2fe-b6f0-8d8b-5809759afd19/WALs/fadb2fe0ba32,43811,1732422333705/fadb2fe0ba32%2C43811%2C1732422333705.1732422335221, exclude list is [], retry=0 2024-11-24T04:25:35,267 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44781,DS-54254ac8-0fae-43f7-ab06-53faa0e05dd7,DISK] 2024-11-24T04:25:35,267 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36069,DS-11c8f594-4fc2-47e4-b222-092ec50eb991,DISK] 2024-11-24T04:25:35,268 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34457,DS-20bb5930-23a3-441d-bdb2-015c0d56b2ec,DISK] 2024-11-24T04:25:35,294 INFO [RS:0;fadb2fe0ba32:35977 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/33c38499-d2fe-b6f0-8d8b-5809759afd19/WALs/fadb2fe0ba32,35977,1732422333601/fadb2fe0ba32%2C35977%2C1732422333601.1732422335220 2024-11-24T04:25:35,298 INFO [RS:2;fadb2fe0ba32:45209 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/33c38499-d2fe-b6f0-8d8b-5809759afd19/WALs/fadb2fe0ba32,45209,1732422333751/fadb2fe0ba32%2C45209%2C1732422333751.1732422335218 2024-11-24T04:25:35,301 DEBUG [RS:0;fadb2fe0ba32:35977 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41009:41009),(127.0.0.1/127.0.0.1:35691:35691),(127.0.0.1/127.0.0.1:33347:33347)] 2024-11-24T04:25:35,306 DEBUG [RS:2;fadb2fe0ba32:45209 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41009:41009),(127.0.0.1/127.0.0.1:33347:33347),(127.0.0.1/127.0.0.1:35691:35691)] 2024-11-24T04:25:35,307 INFO [RS:1;fadb2fe0ba32:43811 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/33c38499-d2fe-b6f0-8d8b-5809759afd19/WALs/fadb2fe0ba32,43811,1732422333705/fadb2fe0ba32%2C43811%2C1732422333705.1732422335221 2024-11-24T04:25:35,308 DEBUG [RS:1;fadb2fe0ba32:43811 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35691:35691),(127.0.0.1/127.0.0.1:33347:33347),(127.0.0.1/127.0.0.1:41009:41009)] 2024-11-24T04:25:35,330 DEBUG [fadb2fe0ba32:33969 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=3, allServersCount=3 2024-11-24T04:25:35,339 DEBUG [fadb2fe0ba32:33969 {}] balancer.BalancerClusterState(204): Hosts are {fadb2fe0ba32=0} racks are {/default-rack=0} 2024-11-24T04:25:35,345 DEBUG [fadb2fe0ba32:33969 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-24T04:25:35,346 DEBUG [fadb2fe0ba32:33969 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-24T04:25:35,346 DEBUG [fadb2fe0ba32:33969 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-11-24T04:25:35,346 DEBUG [fadb2fe0ba32:33969 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-24T04:25:35,346 DEBUG [fadb2fe0ba32:33969 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-24T04:25:35,346 DEBUG [fadb2fe0ba32:33969 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-11-24T04:25:35,346 INFO [fadb2fe0ba32:33969 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-24T04:25:35,346 INFO [fadb2fe0ba32:33969 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-24T04:25:35,346 INFO [fadb2fe0ba32:33969 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-11-24T04:25:35,346 DEBUG [fadb2fe0ba32:33969 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-24T04:25:35,353 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=fadb2fe0ba32,35977,1732422333601 2024-11-24T04:25:35,360 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as fadb2fe0ba32,35977,1732422333601, state=OPENING 2024-11-24T04:25:35,365 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-24T04:25:35,367 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43811-0x1014838d9450002, quorum=127.0.0.1:62017, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T04:25:35,367 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45209-0x1014838d9450003, quorum=127.0.0.1:62017, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T04:25:35,367 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35977-0x1014838d9450001, quorum=127.0.0.1:62017, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T04:25:35,367 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33969-0x1014838d9450000, quorum=127.0.0.1:62017, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T04:25:35,368 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T04:25:35,368 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T04:25:35,368 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T04:25:35,368 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T04:25:35,370 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-24T04:25:35,372 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=fadb2fe0ba32,35977,1732422333601}] 2024-11-24T04:25:35,548 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-24T04:25:35,551 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54301, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-24T04:25:35,563 INFO [RS_OPEN_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-24T04:25:35,564 INFO [RS_OPEN_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T04:25:35,564 INFO [RS_OPEN_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-24T04:25:35,567 INFO [RS_OPEN_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=fadb2fe0ba32%2C35977%2C1732422333601.meta, suffix=.meta, logDir=hdfs://localhost:44611/user/jenkins/test-data/33c38499-d2fe-b6f0-8d8b-5809759afd19/WALs/fadb2fe0ba32,35977,1732422333601, archiveDir=hdfs://localhost:44611/user/jenkins/test-data/33c38499-d2fe-b6f0-8d8b-5809759afd19/oldWALs, maxLogs=32 2024-11-24T04:25:35,584 DEBUG [RS_OPEN_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/33c38499-d2fe-b6f0-8d8b-5809759afd19/WALs/fadb2fe0ba32,35977,1732422333601/fadb2fe0ba32%2C35977%2C1732422333601.meta.1732422335569.meta, exclude list is [], retry=0 2024-11-24T04:25:35,588 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34457,DS-20bb5930-23a3-441d-bdb2-015c0d56b2ec,DISK] 2024-11-24T04:25:35,588 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44781,DS-54254ac8-0fae-43f7-ab06-53faa0e05dd7,DISK] 2024-11-24T04:25:35,588 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36069,DS-11c8f594-4fc2-47e4-b222-092ec50eb991,DISK] 2024-11-24T04:25:35,591 INFO [RS_OPEN_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/33c38499-d2fe-b6f0-8d8b-5809759afd19/WALs/fadb2fe0ba32,35977,1732422333601/fadb2fe0ba32%2C35977%2C1732422333601.meta.1732422335569.meta 2024-11-24T04:25:35,592 DEBUG [RS_OPEN_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33347:33347),(127.0.0.1/127.0.0.1:41009:41009),(127.0.0.1/127.0.0.1:35691:35691)] 2024-11-24T04:25:35,593 DEBUG [RS_OPEN_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-24T04:25:35,594 DEBUG [RS_OPEN_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-24T04:25:35,597 DEBUG [RS_OPEN_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-24T04:25:35,602 INFO [RS_OPEN_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-24T04:25:35,606 DEBUG [RS_OPEN_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-24T04:25:35,607 DEBUG [RS_OPEN_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T04:25:35,607 DEBUG [RS_OPEN_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-24T04:25:35,607 DEBUG [RS_OPEN_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-24T04:25:35,610 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-24T04:25:35,612 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-24T04:25:35,612 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:25:35,613 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T04:25:35,613 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-24T04:25:35,614 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-24T04:25:35,614 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:25:35,615 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T04:25:35,615 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-24T04:25:35,616 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-24T04:25:35,616 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:25:35,617 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T04:25:35,617 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-24T04:25:35,618 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-24T04:25:35,619 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:25:35,619 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T04:25:35,619 DEBUG [RS_OPEN_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-24T04:25:35,621 DEBUG [RS_OPEN_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44611/user/jenkins/test-data/33c38499-d2fe-b6f0-8d8b-5809759afd19/data/hbase/meta/1588230740 2024-11-24T04:25:35,623 DEBUG [RS_OPEN_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44611/user/jenkins/test-data/33c38499-d2fe-b6f0-8d8b-5809759afd19/data/hbase/meta/1588230740 2024-11-24T04:25:35,625 DEBUG [RS_OPEN_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-24T04:25:35,625 DEBUG [RS_OPEN_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-24T04:25:35,626 DEBUG [RS_OPEN_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-24T04:25:35,628 DEBUG [RS_OPEN_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-24T04:25:35,630 INFO [RS_OPEN_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67370754, jitterRate=0.0039024651050567627}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-24T04:25:35,630 DEBUG [RS_OPEN_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-24T04:25:35,632 DEBUG [RS_OPEN_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732422335608Writing region info on filesystem at 1732422335608Initializing all the Stores at 1732422335610 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732422335610Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732422335610Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732422335610Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732422335610Cleaning up temporary data from old regions at 1732422335625 (+15 ms)Running coprocessor post-open hooks at 1732422335630 (+5 ms)Region opened successfully at 1732422335632 (+2 ms) 2024-11-24T04:25:35,639 INFO [RS_OPEN_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732422335539 2024-11-24T04:25:35,651 DEBUG [RS_OPEN_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-24T04:25:35,651 INFO [RS_OPEN_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-24T04:25:35,653 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=fadb2fe0ba32,35977,1732422333601 2024-11-24T04:25:35,655 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as fadb2fe0ba32,35977,1732422333601, state=OPEN 2024-11-24T04:25:35,659 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43811-0x1014838d9450002, quorum=127.0.0.1:62017, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T04:25:35,659 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45209-0x1014838d9450003, quorum=127.0.0.1:62017, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T04:25:35,659 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33969-0x1014838d9450000, quorum=127.0.0.1:62017, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T04:25:35,659 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35977-0x1014838d9450001, quorum=127.0.0.1:62017, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T04:25:35,659 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T04:25:35,659 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T04:25:35,659 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T04:25:35,659 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T04:25:35,659 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=fadb2fe0ba32,35977,1732422333601 2024-11-24T04:25:35,665 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-24T04:25:35,665 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=fadb2fe0ba32,35977,1732422333601 in 288 msec 2024-11-24T04:25:35,671 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-24T04:25:35,671 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 753 msec 2024-11-24T04:25:35,672 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-24T04:25:35,672 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-24T04:25:35,693 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T04:25:35,694 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fadb2fe0ba32,35977,1732422333601, seqNum=-1] 2024-11-24T04:25:35,716 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T04:25:35,718 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56103, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T04:25:35,739 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.0940 sec 2024-11-24T04:25:35,740 INFO [master/fadb2fe0ba32:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732422335740, completionTime=-1 2024-11-24T04:25:35,743 INFO [master/fadb2fe0ba32:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=3; waited=0ms, expected min=3 server(s), max=3 server(s), master is running 2024-11-24T04:25:35,743 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-24T04:25:35,771 INFO [master/fadb2fe0ba32:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=3 2024-11-24T04:25:35,771 INFO [master/fadb2fe0ba32:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732422395771 2024-11-24T04:25:35,771 INFO [master/fadb2fe0ba32:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732422455771 2024-11-24T04:25:35,771 INFO [master/fadb2fe0ba32:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 28 msec 2024-11-24T04:25:35,773 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-11-24T04:25:35,800 INFO [master/fadb2fe0ba32:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=fadb2fe0ba32,33969,1732422332893-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T04:25:35,801 INFO [master/fadb2fe0ba32:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=fadb2fe0ba32,33969,1732422332893-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T04:25:35,801 INFO [master/fadb2fe0ba32:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=fadb2fe0ba32,33969,1732422332893-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T04:25:35,802 INFO [master/fadb2fe0ba32:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-fadb2fe0ba32:33969, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T04:25:35,803 INFO [master/fadb2fe0ba32:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-24T04:25:35,803 INFO [master/fadb2fe0ba32:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-24T04:25:35,812 DEBUG [master/fadb2fe0ba32:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-24T04:25:35,839 INFO [master/fadb2fe0ba32:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 2.008sec 2024-11-24T04:25:35,841 INFO [master/fadb2fe0ba32:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-24T04:25:35,842 INFO [master/fadb2fe0ba32:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-24T04:25:35,843 INFO [master/fadb2fe0ba32:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-24T04:25:35,844 INFO [master/fadb2fe0ba32:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-24T04:25:35,844 INFO [master/fadb2fe0ba32:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-24T04:25:35,845 INFO [master/fadb2fe0ba32:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=fadb2fe0ba32,33969,1732422332893-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-24T04:25:35,845 INFO [master/fadb2fe0ba32:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=fadb2fe0ba32,33969,1732422332893-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-24T04:25:35,850 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-24T04:25:35,851 INFO [master/fadb2fe0ba32:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-24T04:25:35,851 INFO [master/fadb2fe0ba32:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=fadb2fe0ba32,33969,1732422332893-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T04:25:35,908 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3df89a6f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T04:25:35,908 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fadb2fe0ba32,33969,-1 for getting cluster id 2024-11-24T04:25:35,911 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T04:25:35,920 DEBUG [HMaster-EventLoopGroup-2-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '19ccd069-e1d9-4899-ace9-8d275717a5ba' 2024-11-24T04:25:35,923 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T04:25:35,924 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "19ccd069-e1d9-4899-ace9-8d275717a5ba" 2024-11-24T04:25:35,926 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@66359c2a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T04:25:35,926 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fadb2fe0ba32,33969,-1] 2024-11-24T04:25:35,929 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T04:25:35,931 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T04:25:35,932 INFO [HMaster-EventLoopGroup-2-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43942, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T04:25:35,934 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@57c70c1e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T04:25:35,935 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T04:25:35,941 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fadb2fe0ba32,35977,1732422333601, seqNum=-1] 2024-11-24T04:25:35,942 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T04:25:35,944 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48946, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T04:25:35,965 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=fadb2fe0ba32,33969,1732422332893 2024-11-24T04:25:35,966 INFO [Time-limited test {}] wal.AbstractTestWALReplay(147): hbase.rootdir=hdfs://localhost:44611/hbase 2024-11-24T04:25:35,980 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayValueCompression#testNameConflictWhenSplit0 Thread=364, OpenFileDescriptor=603, MaxFileDescriptor=1048576, SystemLoadAverage=205, ProcessCount=11, AvailableMemoryMB=6601 2024-11-24T04:25:35,999 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T04:25:36,002 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T04:25:36,003 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-24T04:25:36,007 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-42545130, suffix=, logDir=hdfs://localhost:44611/hbase/WALs/hregion-42545130, archiveDir=hdfs://localhost:44611/hbase/oldWALs, maxLogs=32 2024-11-24T04:25:36,022 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-42545130/hregion-42545130.1732422336008, exclude list is [], retry=0 2024-11-24T04:25:36,026 DEBUG [AsyncFSWAL-8-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36069,DS-11c8f594-4fc2-47e4-b222-092ec50eb991,DISK] 2024-11-24T04:25:36,026 DEBUG [AsyncFSWAL-8-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34457,DS-20bb5930-23a3-441d-bdb2-015c0d56b2ec,DISK] 2024-11-24T04:25:36,027 DEBUG [AsyncFSWAL-8-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44781,DS-54254ac8-0fae-43f7-ab06-53faa0e05dd7,DISK] 2024-11-24T04:25:36,030 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-42545130/hregion-42545130.1732422336008 2024-11-24T04:25:36,030 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41009:41009),(127.0.0.1/127.0.0.1:33347:33347),(127.0.0.1/127.0.0.1:35691:35691)] 2024-11-24T04:25:36,031 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 505d196bad3873da59b939569d87b6e1, NAME => 'testReplayEditsWrittenIntoWAL,,1732422336000.505d196bad3873da59b939569d87b6e1.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenIntoWAL', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44611/hbase 2024-11-24T04:25:36,043 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34457 is added to blk_1073741838_1014 (size=64) 2024-11-24T04:25:36,044 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44781 is added to blk_1073741838_1014 (size=64) 2024-11-24T04:25:36,044 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36069 is added to blk_1073741838_1014 (size=64) 2024-11-24T04:25:36,045 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1732422336000.505d196bad3873da59b939569d87b6e1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T04:25:36,047 INFO [StoreOpener-505d196bad3873da59b939569d87b6e1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 505d196bad3873da59b939569d87b6e1 2024-11-24T04:25:36,049 INFO [StoreOpener-505d196bad3873da59b939569d87b6e1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 505d196bad3873da59b939569d87b6e1 columnFamilyName a 2024-11-24T04:25:36,049 DEBUG [StoreOpener-505d196bad3873da59b939569d87b6e1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:25:36,050 INFO [StoreOpener-505d196bad3873da59b939569d87b6e1-1 {}] regionserver.HStore(327): Store=505d196bad3873da59b939569d87b6e1/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:25:36,050 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 505d196bad3873da59b939569d87b6e1 2024-11-24T04:25:36,052 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44611/hbase/data/default/testReplayEditsWrittenIntoWAL/505d196bad3873da59b939569d87b6e1 2024-11-24T04:25:36,052 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44611/hbase/data/default/testReplayEditsWrittenIntoWAL/505d196bad3873da59b939569d87b6e1 2024-11-24T04:25:36,053 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 505d196bad3873da59b939569d87b6e1 2024-11-24T04:25:36,053 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 505d196bad3873da59b939569d87b6e1 2024-11-24T04:25:36,057 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 505d196bad3873da59b939569d87b6e1 2024-11-24T04:25:36,060 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44611/hbase/data/default/testReplayEditsWrittenIntoWAL/505d196bad3873da59b939569d87b6e1/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T04:25:36,061 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 505d196bad3873da59b939569d87b6e1; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67280729, jitterRate=0.0025609880685806274}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-24T04:25:36,062 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 505d196bad3873da59b939569d87b6e1: Writing region info on filesystem at 1732422336045Initializing all the Stores at 1732422336046 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732422336047 (+1 ms)Cleaning up temporary data from old regions at 1732422336053 (+6 ms)Region opened successfully at 1732422336062 (+9 ms) 2024-11-24T04:25:36,062 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 505d196bad3873da59b939569d87b6e1, disabling compactions & flushes 2024-11-24T04:25:36,062 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1732422336000.505d196bad3873da59b939569d87b6e1. 2024-11-24T04:25:36,062 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1732422336000.505d196bad3873da59b939569d87b6e1. 2024-11-24T04:25:36,062 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1732422336000.505d196bad3873da59b939569d87b6e1. after waiting 0 ms 2024-11-24T04:25:36,062 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1732422336000.505d196bad3873da59b939569d87b6e1. 2024-11-24T04:25:36,063 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1732422336000.505d196bad3873da59b939569d87b6e1. 2024-11-24T04:25:36,063 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 505d196bad3873da59b939569d87b6e1: Waiting for close lock at 1732422336062Disabling compacts and flushes for region at 1732422336062Disabling writes for close at 1732422336062Writing region close event to WAL at 1732422336063 (+1 ms)Closed at 1732422336063 2024-11-24T04:25:36,068 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44781 is added to blk_1073741837_1013 (size=95) 2024-11-24T04:25:36,068 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34457 is added to blk_1073741837_1013 (size=95) 2024-11-24T04:25:36,068 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36069 is added to blk_1073741837_1013 (size=95) 2024-11-24T04:25:36,075 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-24T04:25:36,075 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-42545130:(num 1732422336008) 2024-11-24T04:25:36,077 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-11-24T04:25:36,092 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36069 is added to blk_1073741839_1015 (size=320) 2024-11-24T04:25:36,092 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44781 is added to blk_1073741839_1015 (size=320) 2024-11-24T04:25:36,093 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34457 is added to blk_1073741839_1015 (size=320) 2024-11-24T04:25:36,094 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-11-24T04:25:36,105 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36069 is added to blk_1073741840_1016 (size=253) 2024-11-24T04:25:36,105 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44781 is added to blk_1073741840_1016 (size=253) 2024-11-24T04:25:36,106 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34457 is added to blk_1073741840_1016 (size=253) 2024-11-24T04:25:36,127 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:44611/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732422335998/wal-1, size=320 (320bytes) 2024-11-24T04:25:36,128 DEBUG [Time-limited test {}] util.RecoverLeaseFSUtils(47): Initialize RecoverLeaseFSUtils 2024-11-24T04:25:36,128 DEBUG [Time-limited test {}] util.RecoverLeaseFSUtils(59): set recoverLeaseMethod to org.apache.hadoop.fs.LeaseRecoverable.recoverLease() 2024-11-24T04:25:36,128 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:44611/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732422335998/wal-1 2024-11-24T04:25:36,134 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:44611/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732422335998/wal-1 after 3ms 2024-11-24T04:25:36,141 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:44611/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732422335998/wal-1: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-24T04:25:36,142 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:44611/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732422335998/wal-1 took 16ms 2024-11-24T04:25:36,154 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:44611/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732422335998/wal-1 so closing down 2024-11-24T04:25:36,154 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-24T04:25:36,157 INFO [Time-limited test-Writer-1 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal-1.temp 2024-11-24T04:25:36,158 INFO [Time-limited test-Writer-1 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:44611/hbase/data/default/testReplayEditsWrittenIntoWAL/505d196bad3873da59b939569d87b6e1/recovered.edits/0000000000000000001-wal-1.temp 2024-11-24T04:25:36,160 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-24T04:25:36,169 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44781 is added to blk_1073741841_1017 (size=320) 2024-11-24T04:25:36,169 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36069 is added to blk_1073741841_1017 (size=320) 2024-11-24T04:25:36,170 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34457 is added to blk_1073741841_1017 (size=320) 2024-11-24T04:25:36,172 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:44611/hbase/data/default/testReplayEditsWrittenIntoWAL/505d196bad3873da59b939569d87b6e1/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) 2024-11-24T04:25:36,174 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:44611/hbase/data/default/testReplayEditsWrittenIntoWAL/505d196bad3873da59b939569d87b6e1/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:44611/hbase/data/default/testReplayEditsWrittenIntoWAL/505d196bad3873da59b939569d87b6e1/recovered.edits/0000000000000000002 2024-11-24T04:25:36,178 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 2 edits across 1 Regions in 30 ms; skipped=0; WAL=hdfs://localhost:44611/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732422335998/wal-1, size=320, length=320, corrupted=false, cancelled=false 2024-11-24T04:25:36,178 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:44611/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732422335998/wal-1, journal: Splitting hdfs://localhost:44611/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732422335998/wal-1, size=320 (320bytes) at 1732422336127Finishing writing output for hdfs://localhost:44611/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732422335998/wal-1 so closing down at 1732422336154 (+27 ms)Creating recovered edits writer path=hdfs://localhost:44611/hbase/data/default/testReplayEditsWrittenIntoWAL/505d196bad3873da59b939569d87b6e1/recovered.edits/0000000000000000001-wal-1.temp at 1732422336158 (+4 ms)3 split writer threads finished at 1732422336160 (+2 ms)Closed recovered edits writer path=hdfs://localhost:44611/hbase/data/default/testReplayEditsWrittenIntoWAL/505d196bad3873da59b939569d87b6e1/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) at 1732422336172 (+12 ms)Rename recovered edits hdfs://localhost:44611/hbase/data/default/testReplayEditsWrittenIntoWAL/505d196bad3873da59b939569d87b6e1/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:44611/hbase/data/default/testReplayEditsWrittenIntoWAL/505d196bad3873da59b939569d87b6e1/recovered.edits/0000000000000000002 at 1732422336174 (+2 ms)Processed 2 edits across 1 Regions in 30 ms; skipped=0; WAL=hdfs://localhost:44611/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732422335998/wal-1, size=320, length=320, corrupted=false, cancelled=false at 1732422336178 (+4 ms) 2024-11-24T04:25:36,193 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:44611/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732422335998/wal-2, size=253 (253bytes) 2024-11-24T04:25:36,193 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:44611/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732422335998/wal-2 2024-11-24T04:25:36,193 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:44611/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732422335998/wal-2 after 0ms 2024-11-24T04:25:36,196 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:44611/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732422335998/wal-2: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-24T04:25:36,197 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:44611/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732422335998/wal-2 took 4ms 2024-11-24T04:25:36,199 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:44611/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732422335998/wal-2 so closing down 2024-11-24T04:25:36,199 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-24T04:25:36,201 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000002-wal-2.temp 2024-11-24T04:25:36,203 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:44611/hbase/data/default/testReplayEditsWrittenIntoWAL/505d196bad3873da59b939569d87b6e1/recovered.edits/0000000000000000002-wal-2.temp 2024-11-24T04:25:36,203 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-24T04:25:36,211 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36069 is added to blk_1073741842_1018 (size=253) 2024-11-24T04:25:36,211 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34457 is added to blk_1073741842_1018 (size=253) 2024-11-24T04:25:36,212 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44781 is added to blk_1073741842_1018 (size=253) 2024-11-24T04:25:36,213 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:44611/hbase/data/default/testReplayEditsWrittenIntoWAL/505d196bad3873da59b939569d87b6e1/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-11-24T04:25:36,217 DEBUG [split-log-closeStream-pool-0 {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:44611/hbase/data/default/testReplayEditsWrittenIntoWAL/505d196bad3873da59b939569d87b6e1/recovered.edits/0000000000000000002: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-24T04:25:36,220 WARN [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(184): Found existing old edits file and we have less entries. Deleting hdfs://localhost:44611/hbase/data/default/testReplayEditsWrittenIntoWAL/505d196bad3873da59b939569d87b6e1/recovered.edits/0000000000000000002-wal-2.temp, length=253 2024-11-24T04:25:36,222 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 1 edits across 1 Regions in 24 ms; skipped=0; WAL=hdfs://localhost:44611/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732422335998/wal-2, size=253, length=253, corrupted=false, cancelled=false 2024-11-24T04:25:36,222 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:44611/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732422335998/wal-2, journal: Splitting hdfs://localhost:44611/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732422335998/wal-2, size=253 (253bytes) at 1732422336193Finishing writing output for hdfs://localhost:44611/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732422335998/wal-2 so closing down at 1732422336199 (+6 ms)Creating recovered edits writer path=hdfs://localhost:44611/hbase/data/default/testReplayEditsWrittenIntoWAL/505d196bad3873da59b939569d87b6e1/recovered.edits/0000000000000000002-wal-2.temp at 1732422336203 (+4 ms)3 split writer threads finished at 1732422336203Closed recovered edits writer path=hdfs://localhost:44611/hbase/data/default/testReplayEditsWrittenIntoWAL/505d196bad3873da59b939569d87b6e1/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1732422336213 (+10 ms)Processed 1 edits across 1 Regions in 24 ms; skipped=0; WAL=hdfs://localhost:44611/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732422335998/wal-2, size=253, length=253, corrupted=false, cancelled=false at 1732422336222 (+9 ms) 2024-11-24T04:25:36,222 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-24T04:25:36,224 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:44611/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732422335998, archiveDir=hdfs://localhost:44611/hbase/oldWALs, maxLogs=32 2024-11-24T04:25:36,239 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testnameconflictwhensplit0-manual,16010,1732422335998/wal.1732422336225, exclude list is [], retry=0 2024-11-24T04:25:36,243 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36069,DS-11c8f594-4fc2-47e4-b222-092ec50eb991,DISK] 2024-11-24T04:25:36,244 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34457,DS-20bb5930-23a3-441d-bdb2-015c0d56b2ec,DISK] 2024-11-24T04:25:36,244 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44781,DS-54254ac8-0fae-43f7-ab06-53faa0e05dd7,DISK] 2024-11-24T04:25:36,247 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testnameconflictwhensplit0-manual,16010,1732422335998/wal.1732422336225 2024-11-24T04:25:36,248 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41009:41009),(127.0.0.1/127.0.0.1:33347:33347),(127.0.0.1/127.0.0.1:35691:35691)] 2024-11-24T04:25:36,248 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 505d196bad3873da59b939569d87b6e1, NAME => 'testReplayEditsWrittenIntoWAL,,1732422336000.505d196bad3873da59b939569d87b6e1.', STARTKEY => '', ENDKEY => ''} 2024-11-24T04:25:36,248 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1732422336000.505d196bad3873da59b939569d87b6e1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T04:25:36,248 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 505d196bad3873da59b939569d87b6e1 2024-11-24T04:25:36,248 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 505d196bad3873da59b939569d87b6e1 2024-11-24T04:25:36,252 INFO [StoreOpener-505d196bad3873da59b939569d87b6e1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 505d196bad3873da59b939569d87b6e1 2024-11-24T04:25:36,254 INFO [StoreOpener-505d196bad3873da59b939569d87b6e1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 505d196bad3873da59b939569d87b6e1 columnFamilyName a 2024-11-24T04:25:36,254 DEBUG [StoreOpener-505d196bad3873da59b939569d87b6e1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:25:36,256 INFO [StoreOpener-505d196bad3873da59b939569d87b6e1-1 {}] regionserver.HStore(327): Store=505d196bad3873da59b939569d87b6e1/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:25:36,256 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 505d196bad3873da59b939569d87b6e1 2024-11-24T04:25:36,257 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44611/hbase/data/default/testReplayEditsWrittenIntoWAL/505d196bad3873da59b939569d87b6e1 2024-11-24T04:25:36,261 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:44611/hbase/data/default/testReplayEditsWrittenIntoWAL/505d196bad3873da59b939569d87b6e1 2024-11-24T04:25:36,262 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:44611/hbase/data/default/testReplayEditsWrittenIntoWAL/505d196bad3873da59b939569d87b6e1/recovered.edits/0000000000000000002 2024-11-24T04:25:36,266 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:44611/hbase/data/default/testReplayEditsWrittenIntoWAL/505d196bad3873da59b939569d87b6e1/recovered.edits/0000000000000000002: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-24T04:25:36,274 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 2, skipped 0, firstSequenceIdInLog=1, maxSequenceIdInLog=2, path=hdfs://localhost:44611/hbase/data/default/testReplayEditsWrittenIntoWAL/505d196bad3873da59b939569d87b6e1/recovered.edits/0000000000000000002 2024-11-24T04:25:36,279 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 505d196bad3873da59b939569d87b6e1 1/1 column families, dataSize=108 B heapSize=512 B 2024-11-24T04:25:36,336 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44611/hbase/data/default/testReplayEditsWrittenIntoWAL/505d196bad3873da59b939569d87b6e1/.tmp/a/4fdb77be088b49d782c9dce920663ee8 is 58, key is testReplayEditsWrittenIntoWAL/a:1/1732422336075/Put/seqid=0 2024-11-24T04:25:36,350 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34457 is added to blk_1073741844_1020 (size=5170) 2024-11-24T04:25:36,350 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36069 is added to blk_1073741844_1020 (size=5170) 2024-11-24T04:25:36,351 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44781 is added to blk_1073741844_1020 (size=5170) 2024-11-24T04:25:36,352 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=108 B at sequenceid=2 (bloomFilter=true), to=hdfs://localhost:44611/hbase/data/default/testReplayEditsWrittenIntoWAL/505d196bad3873da59b939569d87b6e1/.tmp/a/4fdb77be088b49d782c9dce920663ee8 2024-11-24T04:25:36,400 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44611/hbase/data/default/testReplayEditsWrittenIntoWAL/505d196bad3873da59b939569d87b6e1/.tmp/a/4fdb77be088b49d782c9dce920663ee8 as hdfs://localhost:44611/hbase/data/default/testReplayEditsWrittenIntoWAL/505d196bad3873da59b939569d87b6e1/a/4fdb77be088b49d782c9dce920663ee8 2024-11-24T04:25:36,411 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44611/hbase/data/default/testReplayEditsWrittenIntoWAL/505d196bad3873da59b939569d87b6e1/a/4fdb77be088b49d782c9dce920663ee8, entries=2, sequenceid=2, filesize=5.0 K 2024-11-24T04:25:36,418 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for 505d196bad3873da59b939569d87b6e1 in 137ms, sequenceid=2, compaction requested=false; wal=null 2024-11-24T04:25:36,420 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:44611/hbase/data/default/testReplayEditsWrittenIntoWAL/505d196bad3873da59b939569d87b6e1/recovered.edits/0000000000000000002 2024-11-24T04:25:36,421 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 505d196bad3873da59b939569d87b6e1 2024-11-24T04:25:36,421 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 505d196bad3873da59b939569d87b6e1 2024-11-24T04:25:36,424 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 505d196bad3873da59b939569d87b6e1 2024-11-24T04:25:36,427 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44611/hbase/data/default/testReplayEditsWrittenIntoWAL/505d196bad3873da59b939569d87b6e1/recovered.edits/2.seqid, newMaxSeqId=2, maxSeqId=1 2024-11-24T04:25:36,428 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 505d196bad3873da59b939569d87b6e1; next sequenceid=3; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59192524, jitterRate=-0.11796265840530396}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-24T04:25:36,430 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 505d196bad3873da59b939569d87b6e1: Writing region info on filesystem at 1732422336248Initializing all the Stores at 1732422336250 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732422336250Obtaining lock to block concurrent updates at 1732422336279 (+29 ms)Preparing flush snapshotting stores in 505d196bad3873da59b939569d87b6e1 at 1732422336279Finished memstore snapshotting testReplayEditsWrittenIntoWAL,,1732422336000.505d196bad3873da59b939569d87b6e1., syncing WAL and waiting on mvcc, flushsize=dataSize=108, getHeapSize=496, getOffHeapSize=0, getCellsCount=2 at 1732422336283 (+4 ms)Flushing stores of testReplayEditsWrittenIntoWAL,,1732422336000.505d196bad3873da59b939569d87b6e1. at 1732422336283Flushing 505d196bad3873da59b939569d87b6e1/a: creating writer at 1732422336285 (+2 ms)Flushing 505d196bad3873da59b939569d87b6e1/a: appending metadata at 1732422336326 (+41 ms)Flushing 505d196bad3873da59b939569d87b6e1/a: closing flushed file at 1732422336329 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4a95aeea: reopening flushed file at 1732422336398 (+69 ms)Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for 505d196bad3873da59b939569d87b6e1 in 137ms, sequenceid=2, compaction requested=false; wal=null at 1732422336418 (+20 ms)Cleaning up temporary data from old regions at 1732422336421 (+3 ms)Region opened successfully at 1732422336430 (+9 ms) 2024-11-24T04:25:36,456 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayValueCompression#testNameConflictWhenSplit0 Thread=377 (was 364) Potentially hanging thread: AsyncFSWAL-8-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1575425110_22 at /127.0.0.1:33828 [Receiving block BP-1091946017-172.17.0.2-1732422329789:blk_1073741843_1019] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1091946017-172.17.0.2-1732422329789:blk_1073741843_1019, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: TestAsyncWALReplay-pool-0 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1091946017-172.17.0.2-1732422329789:blk_1073741843_1019, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1575425110_22 at /127.0.0.1:34256 [Waiting for operation #10] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-8-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1575425110_22 at /127.0.0.1:34262 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1575425110_22 at /127.0.0.1:34396 [Receiving block BP-1091946017-172.17.0.2-1732422329789:blk_1073741843_1019] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1091946017-172.17.0.2-1732422329789:blk_1073741843_1019, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1575425110_22 at /127.0.0.1:33800 [Waiting for operation #5] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-8-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-0-hdfs://localhost:44611/hbase-prefix:default java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1575425110_22 at /127.0.0.1:34262 [Receiving block BP-1091946017-172.17.0.2-1732422329789:blk_1073741843_1019] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=703 (was 603) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=205 (was 205), ProcessCount=11 (was 11), AvailableMemoryMB=6585 (was 6601) 2024-11-24T04:25:36,467 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayValueCompression#testNameConflictWhenSplit1 Thread=377, OpenFileDescriptor=703, MaxFileDescriptor=1048576, SystemLoadAverage=205, ProcessCount=11, AvailableMemoryMB=6584 2024-11-24T04:25:36,483 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T04:25:36,485 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T04:25:36,486 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-24T04:25:36,489 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-30780358, suffix=, logDir=hdfs://localhost:44611/hbase/WALs/hregion-30780358, archiveDir=hdfs://localhost:44611/hbase/oldWALs, maxLogs=32 2024-11-24T04:25:36,502 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-30780358/hregion-30780358.1732422336489, exclude list is [], retry=0 2024-11-24T04:25:36,506 DEBUG [AsyncFSWAL-10-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44781,DS-54254ac8-0fae-43f7-ab06-53faa0e05dd7,DISK] 2024-11-24T04:25:36,506 DEBUG [AsyncFSWAL-10-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36069,DS-11c8f594-4fc2-47e4-b222-092ec50eb991,DISK] 2024-11-24T04:25:36,506 DEBUG [AsyncFSWAL-10-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34457,DS-20bb5930-23a3-441d-bdb2-015c0d56b2ec,DISK] 2024-11-24T04:25:36,511 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-30780358/hregion-30780358.1732422336489 2024-11-24T04:25:36,511 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35691:35691),(127.0.0.1/127.0.0.1:41009:41009),(127.0.0.1/127.0.0.1:33347:33347)] 2024-11-24T04:25:36,511 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => e241ebfa9276f84813d35472c6da5f35, NAME => 'testReplayEditsWrittenIntoWAL,,1732422336483.e241ebfa9276f84813d35472c6da5f35.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenIntoWAL', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44611/hbase 2024-11-24T04:25:36,522 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36069 is added to blk_1073741846_1022 (size=64) 2024-11-24T04:25:36,522 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44781 is added to blk_1073741846_1022 (size=64) 2024-11-24T04:25:36,523 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34457 is added to blk_1073741846_1022 (size=64) 2024-11-24T04:25:36,524 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1732422336483.e241ebfa9276f84813d35472c6da5f35.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T04:25:36,525 INFO [StoreOpener-e241ebfa9276f84813d35472c6da5f35-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region e241ebfa9276f84813d35472c6da5f35 2024-11-24T04:25:36,528 INFO [StoreOpener-e241ebfa9276f84813d35472c6da5f35-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e241ebfa9276f84813d35472c6da5f35 columnFamilyName a 2024-11-24T04:25:36,528 DEBUG [StoreOpener-e241ebfa9276f84813d35472c6da5f35-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:25:36,529 INFO [StoreOpener-e241ebfa9276f84813d35472c6da5f35-1 {}] regionserver.HStore(327): Store=e241ebfa9276f84813d35472c6da5f35/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:25:36,529 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for e241ebfa9276f84813d35472c6da5f35 2024-11-24T04:25:36,530 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44611/hbase/data/default/testReplayEditsWrittenIntoWAL/e241ebfa9276f84813d35472c6da5f35 2024-11-24T04:25:36,531 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44611/hbase/data/default/testReplayEditsWrittenIntoWAL/e241ebfa9276f84813d35472c6da5f35 2024-11-24T04:25:36,532 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for e241ebfa9276f84813d35472c6da5f35 2024-11-24T04:25:36,532 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for e241ebfa9276f84813d35472c6da5f35 2024-11-24T04:25:36,535 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for e241ebfa9276f84813d35472c6da5f35 2024-11-24T04:25:36,538 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44611/hbase/data/default/testReplayEditsWrittenIntoWAL/e241ebfa9276f84813d35472c6da5f35/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T04:25:36,539 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened e241ebfa9276f84813d35472c6da5f35; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73500227, jitterRate=0.09523873031139374}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-24T04:25:36,539 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for e241ebfa9276f84813d35472c6da5f35: Writing region info on filesystem at 1732422336524Initializing all the Stores at 1732422336525 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732422336525Cleaning up temporary data from old regions at 1732422336532 (+7 ms)Region opened successfully at 1732422336539 (+7 ms) 2024-11-24T04:25:36,539 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing e241ebfa9276f84813d35472c6da5f35, disabling compactions & flushes 2024-11-24T04:25:36,540 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1732422336483.e241ebfa9276f84813d35472c6da5f35. 2024-11-24T04:25:36,540 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1732422336483.e241ebfa9276f84813d35472c6da5f35. 2024-11-24T04:25:36,540 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1732422336483.e241ebfa9276f84813d35472c6da5f35. after waiting 0 ms 2024-11-24T04:25:36,540 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1732422336483.e241ebfa9276f84813d35472c6da5f35. 2024-11-24T04:25:36,540 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1732422336483.e241ebfa9276f84813d35472c6da5f35. 2024-11-24T04:25:36,540 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for e241ebfa9276f84813d35472c6da5f35: Waiting for close lock at 1732422336539Disabling compacts and flushes for region at 1732422336539Disabling writes for close at 1732422336540 (+1 ms)Writing region close event to WAL at 1732422336540Closed at 1732422336540 2024-11-24T04:25:36,544 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34457 is added to blk_1073741845_1021 (size=95) 2024-11-24T04:25:36,544 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36069 is added to blk_1073741845_1021 (size=95) 2024-11-24T04:25:36,545 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44781 is added to blk_1073741845_1021 (size=95) 2024-11-24T04:25:36,548 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-24T04:25:36,548 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-30780358:(num 1732422336489) 2024-11-24T04:25:36,549 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-11-24T04:25:36,562 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36069 is added to blk_1073741847_1023 (size=320) 2024-11-24T04:25:36,562 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34457 is added to blk_1073741847_1023 (size=320) 2024-11-24T04:25:36,562 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44781 is added to blk_1073741847_1023 (size=320) 2024-11-24T04:25:36,564 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-11-24T04:25:36,574 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44781 is added to blk_1073741848_1024 (size=253) 2024-11-24T04:25:36,575 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36069 is added to blk_1073741848_1024 (size=253) 2024-11-24T04:25:36,575 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34457 is added to blk_1073741848_1024 (size=253) 2024-11-24T04:25:36,592 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:44611/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732422336482/wal-2, size=253 (253bytes) 2024-11-24T04:25:36,592 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:44611/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732422336482/wal-2 2024-11-24T04:25:36,593 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:44611/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732422336482/wal-2 after 1ms 2024-11-24T04:25:36,596 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:44611/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732422336482/wal-2: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-24T04:25:36,596 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:44611/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732422336482/wal-2 took 4ms 2024-11-24T04:25:36,598 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:44611/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732422336482/wal-2 so closing down 2024-11-24T04:25:36,598 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-24T04:25:36,600 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000002-wal-2.temp 2024-11-24T04:25:36,602 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:44611/hbase/data/default/testReplayEditsWrittenIntoWAL/e241ebfa9276f84813d35472c6da5f35/recovered.edits/0000000000000000002-wal-2.temp 2024-11-24T04:25:36,602 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-24T04:25:36,610 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36069 is added to blk_1073741849_1025 (size=253) 2024-11-24T04:25:36,610 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34457 is added to blk_1073741849_1025 (size=253) 2024-11-24T04:25:36,611 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44781 is added to blk_1073741849_1025 (size=253) 2024-11-24T04:25:36,612 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:44611/hbase/data/default/testReplayEditsWrittenIntoWAL/e241ebfa9276f84813d35472c6da5f35/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-11-24T04:25:36,614 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:44611/hbase/data/default/testReplayEditsWrittenIntoWAL/e241ebfa9276f84813d35472c6da5f35/recovered.edits/0000000000000000002-wal-2.temp to hdfs://localhost:44611/hbase/data/default/testReplayEditsWrittenIntoWAL/e241ebfa9276f84813d35472c6da5f35/recovered.edits/0000000000000000002 2024-11-24T04:25:36,614 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 1 edits across 1 Regions in 17 ms; skipped=0; WAL=hdfs://localhost:44611/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732422336482/wal-2, size=253, length=253, corrupted=false, cancelled=false 2024-11-24T04:25:36,614 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:44611/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732422336482/wal-2, journal: Splitting hdfs://localhost:44611/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732422336482/wal-2, size=253 (253bytes) at 1732422336592Finishing writing output for hdfs://localhost:44611/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732422336482/wal-2 so closing down at 1732422336598 (+6 ms)Creating recovered edits writer path=hdfs://localhost:44611/hbase/data/default/testReplayEditsWrittenIntoWAL/e241ebfa9276f84813d35472c6da5f35/recovered.edits/0000000000000000002-wal-2.temp at 1732422336602 (+4 ms)3 split writer threads finished at 1732422336602Closed recovered edits writer path=hdfs://localhost:44611/hbase/data/default/testReplayEditsWrittenIntoWAL/e241ebfa9276f84813d35472c6da5f35/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1732422336612 (+10 ms)Rename recovered edits hdfs://localhost:44611/hbase/data/default/testReplayEditsWrittenIntoWAL/e241ebfa9276f84813d35472c6da5f35/recovered.edits/0000000000000000002-wal-2.temp to hdfs://localhost:44611/hbase/data/default/testReplayEditsWrittenIntoWAL/e241ebfa9276f84813d35472c6da5f35/recovered.edits/0000000000000000002 at 1732422336614 (+2 ms)Processed 1 edits across 1 Regions in 17 ms; skipped=0; WAL=hdfs://localhost:44611/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732422336482/wal-2, size=253, length=253, corrupted=false, cancelled=false at 1732422336614 2024-11-24T04:25:36,630 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:44611/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732422336482/wal-1, size=320 (320bytes) 2024-11-24T04:25:36,630 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:44611/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732422336482/wal-1 2024-11-24T04:25:36,630 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:44611/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732422336482/wal-1 after 0ms 2024-11-24T04:25:36,633 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:44611/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732422336482/wal-1: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-24T04:25:36,634 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:44611/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732422336482/wal-1 took 5ms 2024-11-24T04:25:36,636 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:44611/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732422336482/wal-1 so closing down 2024-11-24T04:25:36,636 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-24T04:25:36,638 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal-1.temp 2024-11-24T04:25:36,640 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:44611/hbase/data/default/testReplayEditsWrittenIntoWAL/e241ebfa9276f84813d35472c6da5f35/recovered.edits/0000000000000000001-wal-1.temp 2024-11-24T04:25:36,640 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-24T04:25:36,648 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44781 is added to blk_1073741850_1026 (size=320) 2024-11-24T04:25:36,648 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34457 is added to blk_1073741850_1026 (size=320) 2024-11-24T04:25:36,648 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36069 is added to blk_1073741850_1026 (size=320) 2024-11-24T04:25:36,649 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:44611/hbase/data/default/testReplayEditsWrittenIntoWAL/e241ebfa9276f84813d35472c6da5f35/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) 2024-11-24T04:25:36,652 DEBUG [split-log-closeStream-pool-0 {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:44611/hbase/data/default/testReplayEditsWrittenIntoWAL/e241ebfa9276f84813d35472c6da5f35/recovered.edits/0000000000000000002: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-24T04:25:36,654 WARN [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(175): Found existing old edits file. It could be the result of a previous failed split attempt or we have duplicated wal entries. Deleting hdfs://localhost:44611/hbase/data/default/testReplayEditsWrittenIntoWAL/e241ebfa9276f84813d35472c6da5f35/recovered.edits/0000000000000000002, length=253 2024-11-24T04:25:36,657 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:44611/hbase/data/default/testReplayEditsWrittenIntoWAL/e241ebfa9276f84813d35472c6da5f35/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:44611/hbase/data/default/testReplayEditsWrittenIntoWAL/e241ebfa9276f84813d35472c6da5f35/recovered.edits/0000000000000000002 2024-11-24T04:25:36,658 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 2 edits across 1 Regions in 23 ms; skipped=0; WAL=hdfs://localhost:44611/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732422336482/wal-1, size=320, length=320, corrupted=false, cancelled=false 2024-11-24T04:25:36,658 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:44611/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732422336482/wal-1, journal: Splitting hdfs://localhost:44611/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732422336482/wal-1, size=320 (320bytes) at 1732422336630Finishing writing output for hdfs://localhost:44611/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732422336482/wal-1 so closing down at 1732422336636 (+6 ms)Creating recovered edits writer path=hdfs://localhost:44611/hbase/data/default/testReplayEditsWrittenIntoWAL/e241ebfa9276f84813d35472c6da5f35/recovered.edits/0000000000000000001-wal-1.temp at 1732422336640 (+4 ms)3 split writer threads finished at 1732422336640Closed recovered edits writer path=hdfs://localhost:44611/hbase/data/default/testReplayEditsWrittenIntoWAL/e241ebfa9276f84813d35472c6da5f35/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) at 1732422336649 (+9 ms)Rename recovered edits hdfs://localhost:44611/hbase/data/default/testReplayEditsWrittenIntoWAL/e241ebfa9276f84813d35472c6da5f35/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:44611/hbase/data/default/testReplayEditsWrittenIntoWAL/e241ebfa9276f84813d35472c6da5f35/recovered.edits/0000000000000000002 at 1732422336657 (+8 ms)Processed 2 edits across 1 Regions in 23 ms; skipped=0; WAL=hdfs://localhost:44611/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732422336482/wal-1, size=320, length=320, corrupted=false, cancelled=false at 1732422336658 (+1 ms) 2024-11-24T04:25:36,658 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-24T04:25:36,660 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:44611/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732422336482, archiveDir=hdfs://localhost:44611/hbase/oldWALs, maxLogs=32 2024-11-24T04:25:36,674 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testnameconflictwhensplit1-manual,16010,1732422336482/wal.1732422336661, exclude list is [], retry=0 2024-11-24T04:25:36,678 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44781,DS-54254ac8-0fae-43f7-ab06-53faa0e05dd7,DISK] 2024-11-24T04:25:36,678 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36069,DS-11c8f594-4fc2-47e4-b222-092ec50eb991,DISK] 2024-11-24T04:25:36,678 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34457,DS-20bb5930-23a3-441d-bdb2-015c0d56b2ec,DISK] 2024-11-24T04:25:36,680 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testnameconflictwhensplit1-manual,16010,1732422336482/wal.1732422336661 2024-11-24T04:25:36,681 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35691:35691),(127.0.0.1/127.0.0.1:41009:41009),(127.0.0.1/127.0.0.1:33347:33347)] 2024-11-24T04:25:36,681 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => e241ebfa9276f84813d35472c6da5f35, NAME => 'testReplayEditsWrittenIntoWAL,,1732422336483.e241ebfa9276f84813d35472c6da5f35.', STARTKEY => '', ENDKEY => ''} 2024-11-24T04:25:36,681 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1732422336483.e241ebfa9276f84813d35472c6da5f35.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T04:25:36,681 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for e241ebfa9276f84813d35472c6da5f35 2024-11-24T04:25:36,681 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for e241ebfa9276f84813d35472c6da5f35 2024-11-24T04:25:36,683 INFO [StoreOpener-e241ebfa9276f84813d35472c6da5f35-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region e241ebfa9276f84813d35472c6da5f35 2024-11-24T04:25:36,685 INFO [StoreOpener-e241ebfa9276f84813d35472c6da5f35-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e241ebfa9276f84813d35472c6da5f35 columnFamilyName a 2024-11-24T04:25:36,686 DEBUG [StoreOpener-e241ebfa9276f84813d35472c6da5f35-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:25:36,686 INFO [StoreOpener-e241ebfa9276f84813d35472c6da5f35-1 {}] regionserver.HStore(327): Store=e241ebfa9276f84813d35472c6da5f35/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:25:36,687 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for e241ebfa9276f84813d35472c6da5f35 2024-11-24T04:25:36,687 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44611/hbase/data/default/testReplayEditsWrittenIntoWAL/e241ebfa9276f84813d35472c6da5f35 2024-11-24T04:25:36,690 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:44611/hbase/data/default/testReplayEditsWrittenIntoWAL/e241ebfa9276f84813d35472c6da5f35 2024-11-24T04:25:36,691 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:44611/hbase/data/default/testReplayEditsWrittenIntoWAL/e241ebfa9276f84813d35472c6da5f35/recovered.edits/0000000000000000002 2024-11-24T04:25:36,694 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:44611/hbase/data/default/testReplayEditsWrittenIntoWAL/e241ebfa9276f84813d35472c6da5f35/recovered.edits/0000000000000000002: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-24T04:25:36,696 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 2, skipped 0, firstSequenceIdInLog=1, maxSequenceIdInLog=2, path=hdfs://localhost:44611/hbase/data/default/testReplayEditsWrittenIntoWAL/e241ebfa9276f84813d35472c6da5f35/recovered.edits/0000000000000000002 2024-11-24T04:25:36,696 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing e241ebfa9276f84813d35472c6da5f35 1/1 column families, dataSize=108 B heapSize=512 B 2024-11-24T04:25:36,713 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44611/hbase/data/default/testReplayEditsWrittenIntoWAL/e241ebfa9276f84813d35472c6da5f35/.tmp/a/fb7f76d03faf42408e8a5e5fc81c9d5d is 58, key is testReplayEditsWrittenIntoWAL/a:1/1732422336548/Put/seqid=0 2024-11-24T04:25:36,720 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34457 is added to blk_1073741852_1028 (size=5170) 2024-11-24T04:25:36,721 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36069 is added to blk_1073741852_1028 (size=5170) 2024-11-24T04:25:36,721 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44781 is added to blk_1073741852_1028 (size=5170) 2024-11-24T04:25:36,722 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=108 B at sequenceid=2 (bloomFilter=true), to=hdfs://localhost:44611/hbase/data/default/testReplayEditsWrittenIntoWAL/e241ebfa9276f84813d35472c6da5f35/.tmp/a/fb7f76d03faf42408e8a5e5fc81c9d5d 2024-11-24T04:25:36,729 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44611/hbase/data/default/testReplayEditsWrittenIntoWAL/e241ebfa9276f84813d35472c6da5f35/.tmp/a/fb7f76d03faf42408e8a5e5fc81c9d5d as hdfs://localhost:44611/hbase/data/default/testReplayEditsWrittenIntoWAL/e241ebfa9276f84813d35472c6da5f35/a/fb7f76d03faf42408e8a5e5fc81c9d5d 2024-11-24T04:25:36,737 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44611/hbase/data/default/testReplayEditsWrittenIntoWAL/e241ebfa9276f84813d35472c6da5f35/a/fb7f76d03faf42408e8a5e5fc81c9d5d, entries=2, sequenceid=2, filesize=5.0 K 2024-11-24T04:25:36,737 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for e241ebfa9276f84813d35472c6da5f35 in 41ms, sequenceid=2, compaction requested=false; wal=null 2024-11-24T04:25:36,738 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:44611/hbase/data/default/testReplayEditsWrittenIntoWAL/e241ebfa9276f84813d35472c6da5f35/recovered.edits/0000000000000000002 2024-11-24T04:25:36,739 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for e241ebfa9276f84813d35472c6da5f35 2024-11-24T04:25:36,739 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for e241ebfa9276f84813d35472c6da5f35 2024-11-24T04:25:36,741 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for e241ebfa9276f84813d35472c6da5f35 2024-11-24T04:25:36,744 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44611/hbase/data/default/testReplayEditsWrittenIntoWAL/e241ebfa9276f84813d35472c6da5f35/recovered.edits/2.seqid, newMaxSeqId=2, maxSeqId=1 2024-11-24T04:25:36,745 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened e241ebfa9276f84813d35472c6da5f35; next sequenceid=3; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61855095, jitterRate=-0.07828725874423981}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-24T04:25:36,745 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for e241ebfa9276f84813d35472c6da5f35: Writing region info on filesystem at 1732422336681Initializing all the Stores at 1732422336683 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732422336683Obtaining lock to block concurrent updates at 1732422336696 (+13 ms)Preparing flush snapshotting stores in e241ebfa9276f84813d35472c6da5f35 at 1732422336696Finished memstore snapshotting testReplayEditsWrittenIntoWAL,,1732422336483.e241ebfa9276f84813d35472c6da5f35., syncing WAL and waiting on mvcc, flushsize=dataSize=108, getHeapSize=496, getOffHeapSize=0, getCellsCount=2 at 1732422336696Flushing stores of testReplayEditsWrittenIntoWAL,,1732422336483.e241ebfa9276f84813d35472c6da5f35. at 1732422336696Flushing e241ebfa9276f84813d35472c6da5f35/a: creating writer at 1732422336696Flushing e241ebfa9276f84813d35472c6da5f35/a: appending metadata at 1732422336712 (+16 ms)Flushing e241ebfa9276f84813d35472c6da5f35/a: closing flushed file at 1732422336712Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@38c5d4d2: reopening flushed file at 1732422336728 (+16 ms)Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for e241ebfa9276f84813d35472c6da5f35 in 41ms, sequenceid=2, compaction requested=false; wal=null at 1732422336737 (+9 ms)Cleaning up temporary data from old regions at 1732422336739 (+2 ms)Region opened successfully at 1732422336745 (+6 ms) 2024-11-24T04:25:36,763 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayValueCompression#testNameConflictWhenSplit1 Thread=387 (was 377) Potentially hanging thread: AsyncFSWAL-10-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1091946017-172.17.0.2-1732422329789:blk_1073741851_1027, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1091946017-172.17.0.2-1732422329789:blk_1073741851_1027, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-10-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1575425110_22 at /127.0.0.1:33890 [Receiving block BP-1091946017-172.17.0.2-1732422329789:blk_1073741851_1027] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1575425110_22 at /127.0.0.1:34458 [Receiving block BP-1091946017-172.17.0.2-1732422329789:blk_1073741851_1027] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1575425110_22 at /127.0.0.1:34256 [Waiting for operation #12] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-10-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1575425110_22 at /127.0.0.1:34262 [Waiting for operation #13] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1091946017-172.17.0.2-1732422329789:blk_1073741851_1027, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1575425110_22 at /127.0.0.1:33800 [Waiting for operation #12] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1575425110_22 at /127.0.0.1:34370 [Receiving block BP-1091946017-172.17.0.2-1732422329789:blk_1073741851_1027] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=785 (was 703) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=204 (was 205), ProcessCount=11 (was 11), AvailableMemoryMB=6577 (was 6584) 2024-11-24T04:25:36,774 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayValueCompression#testReplayEditsWrittenIntoWAL Thread=387, OpenFileDescriptor=785, MaxFileDescriptor=1048576, SystemLoadAverage=204, ProcessCount=11, AvailableMemoryMB=6577 2024-11-24T04:25:36,790 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T04:25:36,792 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T04:25:36,793 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-24T04:25:36,796 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-79381055, suffix=, logDir=hdfs://localhost:44611/hbase/WALs/hregion-79381055, archiveDir=hdfs://localhost:44611/hbase/oldWALs, maxLogs=32 2024-11-24T04:25:36,810 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-79381055/hregion-79381055.1732422336796, exclude list is [], retry=0 2024-11-24T04:25:36,813 DEBUG [AsyncFSWAL-12-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36069,DS-11c8f594-4fc2-47e4-b222-092ec50eb991,DISK] 2024-11-24T04:25:36,814 DEBUG [AsyncFSWAL-12-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34457,DS-20bb5930-23a3-441d-bdb2-015c0d56b2ec,DISK] 2024-11-24T04:25:36,814 DEBUG [AsyncFSWAL-12-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44781,DS-54254ac8-0fae-43f7-ab06-53faa0e05dd7,DISK] 2024-11-24T04:25:36,817 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-79381055/hregion-79381055.1732422336796 2024-11-24T04:25:36,817 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41009:41009),(127.0.0.1/127.0.0.1:33347:33347),(127.0.0.1/127.0.0.1:35691:35691)] 2024-11-24T04:25:36,818 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 473d2d7be4089869066228eb8c5b8c73, NAME => 'testReplayEditsWrittenIntoWAL,,1732422336790.473d2d7be4089869066228eb8c5b8c73.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenIntoWAL', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44611/hbase 2024-11-24T04:25:36,830 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36069 is added to blk_1073741854_1030 (size=64) 2024-11-24T04:25:36,830 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34457 is added to blk_1073741854_1030 (size=64) 2024-11-24T04:25:36,831 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44781 is added to blk_1073741854_1030 (size=64) 2024-11-24T04:25:36,832 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1732422336790.473d2d7be4089869066228eb8c5b8c73.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T04:25:36,834 INFO [StoreOpener-473d2d7be4089869066228eb8c5b8c73-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 473d2d7be4089869066228eb8c5b8c73 2024-11-24T04:25:36,836 INFO [StoreOpener-473d2d7be4089869066228eb8c5b8c73-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 473d2d7be4089869066228eb8c5b8c73 columnFamilyName a 2024-11-24T04:25:36,836 DEBUG [StoreOpener-473d2d7be4089869066228eb8c5b8c73-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:25:36,837 INFO [StoreOpener-473d2d7be4089869066228eb8c5b8c73-1 {}] regionserver.HStore(327): Store=473d2d7be4089869066228eb8c5b8c73/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:25:36,837 INFO [StoreOpener-473d2d7be4089869066228eb8c5b8c73-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 473d2d7be4089869066228eb8c5b8c73 2024-11-24T04:25:36,838 INFO [StoreOpener-473d2d7be4089869066228eb8c5b8c73-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 473d2d7be4089869066228eb8c5b8c73 columnFamilyName b 2024-11-24T04:25:36,839 DEBUG [StoreOpener-473d2d7be4089869066228eb8c5b8c73-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:25:36,839 INFO [StoreOpener-473d2d7be4089869066228eb8c5b8c73-1 {}] regionserver.HStore(327): Store=473d2d7be4089869066228eb8c5b8c73/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:25:36,839 INFO [StoreOpener-473d2d7be4089869066228eb8c5b8c73-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 473d2d7be4089869066228eb8c5b8c73 2024-11-24T04:25:36,841 INFO [StoreOpener-473d2d7be4089869066228eb8c5b8c73-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 473d2d7be4089869066228eb8c5b8c73 columnFamilyName c 2024-11-24T04:25:36,841 DEBUG [StoreOpener-473d2d7be4089869066228eb8c5b8c73-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:25:36,842 INFO [StoreOpener-473d2d7be4089869066228eb8c5b8c73-1 {}] regionserver.HStore(327): Store=473d2d7be4089869066228eb8c5b8c73/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:25:36,842 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 473d2d7be4089869066228eb8c5b8c73 2024-11-24T04:25:36,843 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44611/hbase/data/default/testReplayEditsWrittenIntoWAL/473d2d7be4089869066228eb8c5b8c73 2024-11-24T04:25:36,843 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44611/hbase/data/default/testReplayEditsWrittenIntoWAL/473d2d7be4089869066228eb8c5b8c73 2024-11-24T04:25:36,845 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 473d2d7be4089869066228eb8c5b8c73 2024-11-24T04:25:36,845 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 473d2d7be4089869066228eb8c5b8c73 2024-11-24T04:25:36,845 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenIntoWAL descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-24T04:25:36,847 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 473d2d7be4089869066228eb8c5b8c73 2024-11-24T04:25:36,850 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44611/hbase/data/default/testReplayEditsWrittenIntoWAL/473d2d7be4089869066228eb8c5b8c73/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T04:25:36,850 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 473d2d7be4089869066228eb8c5b8c73; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60870344, jitterRate=-0.09296119213104248}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-24T04:25:36,851 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 473d2d7be4089869066228eb8c5b8c73: Writing region info on filesystem at 1732422336832Initializing all the Stores at 1732422336833 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732422336833Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732422336833Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732422336833Cleaning up temporary data from old regions at 1732422336845 (+12 ms)Region opened successfully at 1732422336850 (+5 ms) 2024-11-24T04:25:36,851 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 473d2d7be4089869066228eb8c5b8c73, disabling compactions & flushes 2024-11-24T04:25:36,851 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1732422336790.473d2d7be4089869066228eb8c5b8c73. 2024-11-24T04:25:36,851 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1732422336790.473d2d7be4089869066228eb8c5b8c73. 2024-11-24T04:25:36,851 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1732422336790.473d2d7be4089869066228eb8c5b8c73. after waiting 0 ms 2024-11-24T04:25:36,851 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1732422336790.473d2d7be4089869066228eb8c5b8c73. 2024-11-24T04:25:36,851 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1732422336790.473d2d7be4089869066228eb8c5b8c73. 2024-11-24T04:25:36,852 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 473d2d7be4089869066228eb8c5b8c73: Waiting for close lock at 1732422336851Disabling compacts and flushes for region at 1732422336851Disabling writes for close at 1732422336851Writing region close event to WAL at 1732422336851Closed at 1732422336851 2024-11-24T04:25:36,855 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44781 is added to blk_1073741853_1029 (size=95) 2024-11-24T04:25:36,855 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36069 is added to blk_1073741853_1029 (size=95) 2024-11-24T04:25:36,855 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34457 is added to blk_1073741853_1029 (size=95) 2024-11-24T04:25:36,859 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-24T04:25:36,859 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-79381055:(num 1732422336796) 2024-11-24T04:25:36,859 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-24T04:25:36,861 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:44611/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732422336789, archiveDir=hdfs://localhost:44611/hbase/oldWALs, maxLogs=32 2024-11-24T04:25:36,876 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732422336789/wal.1732422336862, exclude list is [], retry=0 2024-11-24T04:25:36,879 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34457,DS-20bb5930-23a3-441d-bdb2-015c0d56b2ec,DISK] 2024-11-24T04:25:36,879 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44781,DS-54254ac8-0fae-43f7-ab06-53faa0e05dd7,DISK] 2024-11-24T04:25:36,880 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36069,DS-11c8f594-4fc2-47e4-b222-092ec50eb991,DISK] 2024-11-24T04:25:36,883 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732422336789/wal.1732422336862 2024-11-24T04:25:36,884 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33347:33347),(127.0.0.1/127.0.0.1:35691:35691),(127.0.0.1/127.0.0.1:41009:41009)] 2024-11-24T04:25:37,096 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:44611/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732422336789/wal.1732422336862, size=0 (0bytes) 2024-11-24T04:25:37,096 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:44611/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732422336789/wal.1732422336862 might be still open, length is 0 2024-11-24T04:25:37,096 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:44611/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732422336789/wal.1732422336862 2024-11-24T04:25:37,097 WARN [IPC Server handler 3 on default port 44611 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732422336789/wal.1732422336862 has not been closed. Lease recovery is in progress. RecoveryId = 1032 for block blk_1073741855_1031 2024-11-24T04:25:37,098 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:44611/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732422336789/wal.1732422336862 after 2ms 2024-11-24T04:25:37,854 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1575425110_22 at /127.0.0.1:33920 [Receiving block BP-1091946017-172.17.0.2-1732422329789:blk_1073741855_1031] {}] datanode.DataXceiver(331): 127.0.0.1:34457:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:33920 dst: /127.0.0.1:34457 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:34457 remote=/127.0.0.1:33920]. Total timeout mills is 60000, 59203 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T04:25:37,855 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1575425110_22 at /127.0.0.1:34494 [Receiving block BP-1091946017-172.17.0.2-1732422329789:blk_1073741855_1031] {}] datanode.DataXceiver(331): 127.0.0.1:36069:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:34494 dst: /127.0.0.1:36069 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T04:25:37,855 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1575425110_22 at /127.0.0.1:34420 [Receiving block BP-1091946017-172.17.0.2-1732422329789:blk_1073741855_1031] {}] datanode.DataXceiver(331): 127.0.0.1:44781:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:34420 dst: /127.0.0.1:44781 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T04:25:37,863 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34457 is added to blk_1073741855_1032 (size=263633) 2024-11-24T04:25:37,865 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44781 is added to blk_1073741855_1032 (size=263633) 2024-11-24T04:25:37,866 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36069 is added to blk_1073741855_1032 (size=263633) 2024-11-24T04:25:41,099 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:44611/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732422336789/wal.1732422336862 after 4003ms 2024-11-24T04:25:41,103 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:44611/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732422336789/wal.1732422336862: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-24T04:25:41,105 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:44611/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732422336789/wal.1732422336862 took 4009ms 2024-11-24T04:25:41,110 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal.1732422336862.temp 2024-11-24T04:25:41,111 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-24T04:25:41,112 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:44611/hbase/data/default/testReplayEditsWrittenIntoWAL/473d2d7be4089869066228eb8c5b8c73/recovered.edits/0000000000000000001-wal.1732422336862.temp 2024-11-24T04:25:41,195 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-24T04:25:41,257 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:44611/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732422336789/wal.1732422336862; continuing. 2024-11-24T04:25:41,257 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:44611/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732422336789/wal.1732422336862 so closing down 2024-11-24T04:25:41,257 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-24T04:25:41,257 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-24T04:25:41,260 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36069 is added to blk_1073741856_1033 (size=263641) 2024-11-24T04:25:41,261 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44781 is added to blk_1073741856_1033 (size=263641) 2024-11-24T04:25:41,261 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34457 is added to blk_1073741856_1033 (size=263641) 2024-11-24T04:25:41,263 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:44611/hbase/data/default/testReplayEditsWrittenIntoWAL/473d2d7be4089869066228eb8c5b8c73/recovered.edits/0000000000000000001-wal.1732422336862.temp (wrote 3002 edits, skipped 0 edits in 72 ms) 2024-11-24T04:25:41,265 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:44611/hbase/data/default/testReplayEditsWrittenIntoWAL/473d2d7be4089869066228eb8c5b8c73/recovered.edits/0000000000000000001-wal.1732422336862.temp to hdfs://localhost:44611/hbase/data/default/testReplayEditsWrittenIntoWAL/473d2d7be4089869066228eb8c5b8c73/recovered.edits/0000000000000003002 2024-11-24T04:25:41,265 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 3002 edits across 1 Regions in 160 ms; skipped=0; WAL=hdfs://localhost:44611/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732422336789/wal.1732422336862, size=0, length=0, corrupted=false, cancelled=false 2024-11-24T04:25:41,265 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:44611/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732422336789/wal.1732422336862, journal: Splitting hdfs://localhost:44611/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732422336789/wal.1732422336862, size=0 (0bytes) at 1732422337096Creating recovered edits writer path=hdfs://localhost:44611/hbase/data/default/testReplayEditsWrittenIntoWAL/473d2d7be4089869066228eb8c5b8c73/recovered.edits/0000000000000000001-wal.1732422336862.temp at 1732422341112 (+4016 ms)Split 1024 edits, skipped 0 edits. at 1732422341203 (+91 ms)Split 2048 edits, skipped 0 edits. at 1732422341233 (+30 ms)Finishing writing output for hdfs://localhost:44611/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732422336789/wal.1732422336862 so closing down at 1732422341257 (+24 ms)3 split writer threads finished at 1732422341257Closed recovered edits writer path=hdfs://localhost:44611/hbase/data/default/testReplayEditsWrittenIntoWAL/473d2d7be4089869066228eb8c5b8c73/recovered.edits/0000000000000000001-wal.1732422336862.temp (wrote 3002 edits, skipped 0 edits in 72 ms) at 1732422341263 (+6 ms)Rename recovered edits hdfs://localhost:44611/hbase/data/default/testReplayEditsWrittenIntoWAL/473d2d7be4089869066228eb8c5b8c73/recovered.edits/0000000000000000001-wal.1732422336862.temp to hdfs://localhost:44611/hbase/data/default/testReplayEditsWrittenIntoWAL/473d2d7be4089869066228eb8c5b8c73/recovered.edits/0000000000000003002 at 1732422341265 (+2 ms)Processed 3002 edits across 1 Regions in 160 ms; skipped=0; WAL=hdfs://localhost:44611/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732422336789/wal.1732422336862, size=0, length=0, corrupted=false, cancelled=false at 1732422341265 2024-11-24T04:25:41,268 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:44611/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732422336789/wal.1732422336862 to hdfs://localhost:44611/hbase/oldWALs/wal.1732422336862 2024-11-24T04:25:41,269 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:44611/hbase/data/default/testReplayEditsWrittenIntoWAL/473d2d7be4089869066228eb8c5b8c73/recovered.edits/0000000000000003002 2024-11-24T04:25:41,269 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-24T04:25:41,272 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:44611/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732422336789, archiveDir=hdfs://localhost:44611/hbase/oldWALs, maxLogs=32 2024-11-24T04:25:41,293 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732422336789/wal.1732422341272, exclude list is [], retry=0 2024-11-24T04:25:41,297 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44781,DS-54254ac8-0fae-43f7-ab06-53faa0e05dd7,DISK] 2024-11-24T04:25:41,298 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34457,DS-20bb5930-23a3-441d-bdb2-015c0d56b2ec,DISK] 2024-11-24T04:25:41,298 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36069,DS-11c8f594-4fc2-47e4-b222-092ec50eb991,DISK] 2024-11-24T04:25:41,301 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732422336789/wal.1732422341272 2024-11-24T04:25:41,301 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35691:35691),(127.0.0.1/127.0.0.1:33347:33347),(127.0.0.1/127.0.0.1:41009:41009)] 2024-11-24T04:25:41,302 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1732422336790.473d2d7be4089869066228eb8c5b8c73.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T04:25:41,306 INFO [StoreOpener-473d2d7be4089869066228eb8c5b8c73-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 473d2d7be4089869066228eb8c5b8c73 2024-11-24T04:25:41,308 INFO [StoreOpener-473d2d7be4089869066228eb8c5b8c73-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 473d2d7be4089869066228eb8c5b8c73 columnFamilyName a 2024-11-24T04:25:41,308 DEBUG [StoreOpener-473d2d7be4089869066228eb8c5b8c73-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:25:41,309 INFO [StoreOpener-473d2d7be4089869066228eb8c5b8c73-1 {}] regionserver.HStore(327): Store=473d2d7be4089869066228eb8c5b8c73/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:25:41,309 INFO [StoreOpener-473d2d7be4089869066228eb8c5b8c73-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 473d2d7be4089869066228eb8c5b8c73 2024-11-24T04:25:41,310 INFO [StoreOpener-473d2d7be4089869066228eb8c5b8c73-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 473d2d7be4089869066228eb8c5b8c73 columnFamilyName b 2024-11-24T04:25:41,310 DEBUG [StoreOpener-473d2d7be4089869066228eb8c5b8c73-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:25:41,310 INFO [StoreOpener-473d2d7be4089869066228eb8c5b8c73-1 {}] regionserver.HStore(327): Store=473d2d7be4089869066228eb8c5b8c73/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:25:41,311 INFO [StoreOpener-473d2d7be4089869066228eb8c5b8c73-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 473d2d7be4089869066228eb8c5b8c73 2024-11-24T04:25:41,311 INFO [StoreOpener-473d2d7be4089869066228eb8c5b8c73-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 473d2d7be4089869066228eb8c5b8c73 columnFamilyName c 2024-11-24T04:25:41,312 DEBUG [StoreOpener-473d2d7be4089869066228eb8c5b8c73-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:25:41,312 INFO [StoreOpener-473d2d7be4089869066228eb8c5b8c73-1 {}] regionserver.HStore(327): Store=473d2d7be4089869066228eb8c5b8c73/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:25:41,312 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 473d2d7be4089869066228eb8c5b8c73 2024-11-24T04:25:41,313 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44611/hbase/data/default/testReplayEditsWrittenIntoWAL/473d2d7be4089869066228eb8c5b8c73 2024-11-24T04:25:41,315 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:44611/hbase/data/default/testReplayEditsWrittenIntoWAL/473d2d7be4089869066228eb8c5b8c73 2024-11-24T04:25:41,316 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:44611/hbase/data/default/testReplayEditsWrittenIntoWAL/473d2d7be4089869066228eb8c5b8c73/recovered.edits/0000000000000003002 2024-11-24T04:25:41,319 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:44611/hbase/data/default/testReplayEditsWrittenIntoWAL/473d2d7be4089869066228eb8c5b8c73/recovered.edits/0000000000000003002: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-24T04:25:41,363 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-11-24T04:25:41,764 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 473d2d7be4089869066228eb8c5b8c73 3/3 column families, dataSize=42.49 KB heapSize=100.11 KB 2024-11-24T04:25:41,800 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44611/hbase/data/default/testReplayEditsWrittenIntoWAL/473d2d7be4089869066228eb8c5b8c73/.tmp/a/b78d29f791a74e4fb793fb558feaf5c0 is 62, key is testReplayEditsWrittenIntoWAL/a:100/1732422336894/Put/seqid=0 2024-11-24T04:25:41,809 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34457 is added to blk_1073741858_1035 (size=50463) 2024-11-24T04:25:41,809 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44781 is added to blk_1073741858_1035 (size=50463) 2024-11-24T04:25:41,809 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36069 is added to blk_1073741858_1035 (size=50463) 2024-11-24T04:25:41,810 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=754 (bloomFilter=true), to=hdfs://localhost:44611/hbase/data/default/testReplayEditsWrittenIntoWAL/473d2d7be4089869066228eb8c5b8c73/.tmp/a/b78d29f791a74e4fb793fb558feaf5c0 2024-11-24T04:25:41,819 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44611/hbase/data/default/testReplayEditsWrittenIntoWAL/473d2d7be4089869066228eb8c5b8c73/.tmp/a/b78d29f791a74e4fb793fb558feaf5c0 as hdfs://localhost:44611/hbase/data/default/testReplayEditsWrittenIntoWAL/473d2d7be4089869066228eb8c5b8c73/a/b78d29f791a74e4fb793fb558feaf5c0 2024-11-24T04:25:41,826 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44611/hbase/data/default/testReplayEditsWrittenIntoWAL/473d2d7be4089869066228eb8c5b8c73/a/b78d29f791a74e4fb793fb558feaf5c0, entries=754, sequenceid=754, filesize=49.3 K 2024-11-24T04:25:41,826 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~42.49 KB/43512, heapSize ~99.59 KB/101984, currentSize=0 B/0 for 473d2d7be4089869066228eb8c5b8c73 in 63ms, sequenceid=754, compaction requested=false; wal=null 2024-11-24T04:25:41,841 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-11-24T04:25:41,841 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 473d2d7be4089869066228eb8c5b8c73 3/3 column families, dataSize=42.49 KB heapSize=100.11 KB 2024-11-24T04:25:41,850 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44611/hbase/data/default/testReplayEditsWrittenIntoWAL/473d2d7be4089869066228eb8c5b8c73/.tmp/a/12f0de56fd0e46e6a267f0bcb2f679f3 is 62, key is testReplayEditsWrittenIntoWAL/a:754/1732422336951/Put/seqid=0 2024-11-24T04:25:41,859 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44781 is added to blk_1073741859_1036 (size=20072) 2024-11-24T04:25:41,859 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36069 is added to blk_1073741859_1036 (size=20072) 2024-11-24T04:25:41,860 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34457 is added to blk_1073741859_1036 (size=20072) 2024-11-24T04:25:41,860 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.93 KB at sequenceid=1508 (bloomFilter=true), to=hdfs://localhost:44611/hbase/data/default/testReplayEditsWrittenIntoWAL/473d2d7be4089869066228eb8c5b8c73/.tmp/a/12f0de56fd0e46e6a267f0bcb2f679f3 2024-11-24T04:25:41,887 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44611/hbase/data/default/testReplayEditsWrittenIntoWAL/473d2d7be4089869066228eb8c5b8c73/.tmp/b/dd5db0f2d9aa4ccc955085097f5c6b40 is 62, key is testReplayEditsWrittenIntoWAL/b:100/1732422336976/Put/seqid=0 2024-11-24T04:25:41,894 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36069 is added to blk_1073741860_1037 (size=35835) 2024-11-24T04:25:41,895 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34457 is added to blk_1073741860_1037 (size=35835) 2024-11-24T04:25:41,895 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44781 is added to blk_1073741860_1037 (size=35835) 2024-11-24T04:25:41,896 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=28.56 KB at sequenceid=1508 (bloomFilter=true), to=hdfs://localhost:44611/hbase/data/default/testReplayEditsWrittenIntoWAL/473d2d7be4089869066228eb8c5b8c73/.tmp/b/dd5db0f2d9aa4ccc955085097f5c6b40 2024-11-24T04:25:41,902 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44611/hbase/data/default/testReplayEditsWrittenIntoWAL/473d2d7be4089869066228eb8c5b8c73/.tmp/a/12f0de56fd0e46e6a267f0bcb2f679f3 as hdfs://localhost:44611/hbase/data/default/testReplayEditsWrittenIntoWAL/473d2d7be4089869066228eb8c5b8c73/a/12f0de56fd0e46e6a267f0bcb2f679f3 2024-11-24T04:25:41,909 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44611/hbase/data/default/testReplayEditsWrittenIntoWAL/473d2d7be4089869066228eb8c5b8c73/a/12f0de56fd0e46e6a267f0bcb2f679f3, entries=246, sequenceid=1508, filesize=19.6 K 2024-11-24T04:25:41,911 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44611/hbase/data/default/testReplayEditsWrittenIntoWAL/473d2d7be4089869066228eb8c5b8c73/.tmp/b/dd5db0f2d9aa4ccc955085097f5c6b40 as hdfs://localhost:44611/hbase/data/default/testReplayEditsWrittenIntoWAL/473d2d7be4089869066228eb8c5b8c73/b/dd5db0f2d9aa4ccc955085097f5c6b40 2024-11-24T04:25:41,917 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44611/hbase/data/default/testReplayEditsWrittenIntoWAL/473d2d7be4089869066228eb8c5b8c73/b/dd5db0f2d9aa4ccc955085097f5c6b40, entries=508, sequenceid=1508, filesize=35.0 K 2024-11-24T04:25:41,918 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~42.49 KB/43512, heapSize ~99.83 KB/102224, currentSize=0 B/0 for 473d2d7be4089869066228eb8c5b8c73 in 76ms, sequenceid=1508, compaction requested=false; wal=null 2024-11-24T04:25:41,928 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-11-24T04:25:41,928 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 473d2d7be4089869066228eb8c5b8c73 3/3 column families, dataSize=42.49 KB heapSize=100.11 KB 2024-11-24T04:25:41,934 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44611/hbase/data/default/testReplayEditsWrittenIntoWAL/473d2d7be4089869066228eb8c5b8c73/.tmp/b/9a96dae5c4d247afa8536e76dc8f1cc8 is 62, key is testReplayEditsWrittenIntoWAL/b:508/1732422336993/Put/seqid=0 2024-11-24T04:25:41,943 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36069 is added to blk_1073741861_1038 (size=35082) 2024-11-24T04:25:41,944 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34457 is added to blk_1073741861_1038 (size=35082) 2024-11-24T04:25:41,944 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44781 is added to blk_1073741861_1038 (size=35082) 2024-11-24T04:25:41,944 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=27.87 KB at sequenceid=2262 (bloomFilter=true), to=hdfs://localhost:44611/hbase/data/default/testReplayEditsWrittenIntoWAL/473d2d7be4089869066228eb8c5b8c73/.tmp/b/9a96dae5c4d247afa8536e76dc8f1cc8 2024-11-24T04:25:41,970 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44611/hbase/data/default/testReplayEditsWrittenIntoWAL/473d2d7be4089869066228eb8c5b8c73/.tmp/c/41e9333045924c948f45d028c9d627ca is 62, key is testReplayEditsWrittenIntoWAL/c:100/1732422337018/Put/seqid=0 2024-11-24T04:25:41,981 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44781 is added to blk_1073741862_1039 (size=20825) 2024-11-24T04:25:41,981 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34457 is added to blk_1073741862_1039 (size=20825) 2024-11-24T04:25:41,981 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36069 is added to blk_1073741862_1039 (size=20825) 2024-11-24T04:25:41,982 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=14.63 KB at sequenceid=2262 (bloomFilter=true), to=hdfs://localhost:44611/hbase/data/default/testReplayEditsWrittenIntoWAL/473d2d7be4089869066228eb8c5b8c73/.tmp/c/41e9333045924c948f45d028c9d627ca 2024-11-24T04:25:41,989 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44611/hbase/data/default/testReplayEditsWrittenIntoWAL/473d2d7be4089869066228eb8c5b8c73/.tmp/b/9a96dae5c4d247afa8536e76dc8f1cc8 as hdfs://localhost:44611/hbase/data/default/testReplayEditsWrittenIntoWAL/473d2d7be4089869066228eb8c5b8c73/b/9a96dae5c4d247afa8536e76dc8f1cc8 2024-11-24T04:25:41,996 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44611/hbase/data/default/testReplayEditsWrittenIntoWAL/473d2d7be4089869066228eb8c5b8c73/b/9a96dae5c4d247afa8536e76dc8f1cc8, entries=492, sequenceid=2262, filesize=34.3 K 2024-11-24T04:25:41,997 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44611/hbase/data/default/testReplayEditsWrittenIntoWAL/473d2d7be4089869066228eb8c5b8c73/.tmp/c/41e9333045924c948f45d028c9d627ca as hdfs://localhost:44611/hbase/data/default/testReplayEditsWrittenIntoWAL/473d2d7be4089869066228eb8c5b8c73/c/41e9333045924c948f45d028c9d627ca 2024-11-24T04:25:42,003 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44611/hbase/data/default/testReplayEditsWrittenIntoWAL/473d2d7be4089869066228eb8c5b8c73/c/41e9333045924c948f45d028c9d627ca, entries=262, sequenceid=2262, filesize=20.3 K 2024-11-24T04:25:42,003 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~42.49 KB/43512, heapSize ~99.83 KB/102224, currentSize=0 B/0 for 473d2d7be4089869066228eb8c5b8c73 in 75ms, sequenceid=2262, compaction requested=false; wal=null 2024-11-24T04:25:42,013 WARN [Time-limited test {}] regionserver.HRegion(5722): No family for cell testReplayEditsWrittenIntoWAL/another family:testReplayEditsWrittenIntoWAL/1732422337055/Put/vlen=29/seqid=0 in region testReplayEditsWrittenIntoWAL,,1732422336790.473d2d7be4089869066228eb8c5b8c73. 2024-11-24T04:25:42,016 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 3001, skipped 1, firstSequenceIdInLog=1, maxSequenceIdInLog=3002, path=hdfs://localhost:44611/hbase/data/default/testReplayEditsWrittenIntoWAL/473d2d7be4089869066228eb8c5b8c73/recovered.edits/0000000000000003002 2024-11-24T04:25:42,016 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-11-24T04:25:42,017 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 473d2d7be4089869066228eb8c5b8c73 3/3 column families, dataSize=41.85 KB heapSize=98.89 KB 2024-11-24T04:25:42,024 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44611/hbase/data/default/testReplayEditsWrittenIntoWAL/473d2d7be4089869066228eb8c5b8c73/.tmp/c/ace72b150b0e4aa9b360f266cd660abc is 62, key is testReplayEditsWrittenIntoWAL/c:262/1732422337024/Put/seqid=0 2024-11-24T04:25:42,032 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36069 is added to blk_1073741863_1040 (size=50301) 2024-11-24T04:25:42,032 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44781 is added to blk_1073741863_1040 (size=50301) 2024-11-24T04:25:42,033 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34457 is added to blk_1073741863_1040 (size=50301) 2024-11-24T04:25:42,033 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=41.85 KB at sequenceid=3002 (bloomFilter=true), to=hdfs://localhost:44611/hbase/data/default/testReplayEditsWrittenIntoWAL/473d2d7be4089869066228eb8c5b8c73/.tmp/c/ace72b150b0e4aa9b360f266cd660abc 2024-11-24T04:25:42,042 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for ace72b150b0e4aa9b360f266cd660abc 2024-11-24T04:25:42,044 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44611/hbase/data/default/testReplayEditsWrittenIntoWAL/473d2d7be4089869066228eb8c5b8c73/.tmp/c/ace72b150b0e4aa9b360f266cd660abc as hdfs://localhost:44611/hbase/data/default/testReplayEditsWrittenIntoWAL/473d2d7be4089869066228eb8c5b8c73/c/ace72b150b0e4aa9b360f266cd660abc 2024-11-24T04:25:42,052 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for ace72b150b0e4aa9b360f266cd660abc 2024-11-24T04:25:42,052 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44611/hbase/data/default/testReplayEditsWrittenIntoWAL/473d2d7be4089869066228eb8c5b8c73/c/ace72b150b0e4aa9b360f266cd660abc, entries=739, sequenceid=3002, filesize=49.1 K 2024-11-24T04:25:42,052 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~41.85 KB/42854, heapSize ~98.38 KB/100736, currentSize=0 B/0 for 473d2d7be4089869066228eb8c5b8c73 in 36ms, sequenceid=3002, compaction requested=false; wal=null 2024-11-24T04:25:42,053 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:44611/hbase/data/default/testReplayEditsWrittenIntoWAL/473d2d7be4089869066228eb8c5b8c73/recovered.edits/0000000000000003002 2024-11-24T04:25:42,055 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 473d2d7be4089869066228eb8c5b8c73 2024-11-24T04:25:42,055 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 473d2d7be4089869066228eb8c5b8c73 2024-11-24T04:25:42,056 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenIntoWAL descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-24T04:25:42,057 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 473d2d7be4089869066228eb8c5b8c73 2024-11-24T04:25:42,060 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44611/hbase/data/default/testReplayEditsWrittenIntoWAL/473d2d7be4089869066228eb8c5b8c73/recovered.edits/3002.seqid, newMaxSeqId=3002, maxSeqId=1 2024-11-24T04:25:42,061 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 473d2d7be4089869066228eb8c5b8c73; next sequenceid=3003; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=204800, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67688208, jitterRate=0.008632898330688477}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-24T04:25:42,061 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 473d2d7be4089869066228eb8c5b8c73: Writing region info on filesystem at 1732422341302Initializing all the Stores at 1732422341303 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732422341303Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732422341306 (+3 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732422341306Cleaning up temporary data from old regions at 1732422342055 (+749 ms)Region opened successfully at 1732422342061 (+6 ms) 2024-11-24T04:25:42,136 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 473d2d7be4089869066228eb8c5b8c73, disabling compactions & flushes 2024-11-24T04:25:42,136 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1732422336790.473d2d7be4089869066228eb8c5b8c73. 2024-11-24T04:25:42,137 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1732422336790.473d2d7be4089869066228eb8c5b8c73. 2024-11-24T04:25:42,137 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1732422336790.473d2d7be4089869066228eb8c5b8c73. after waiting 0 ms 2024-11-24T04:25:42,137 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1732422336790.473d2d7be4089869066228eb8c5b8c73. 2024-11-24T04:25:42,140 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1732422336790.473d2d7be4089869066228eb8c5b8c73. 2024-11-24T04:25:42,141 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 473d2d7be4089869066228eb8c5b8c73: Waiting for close lock at 1732422342136Disabling compacts and flushes for region at 1732422342136Disabling writes for close at 1732422342137 (+1 ms)Writing region close event to WAL at 1732422342140 (+3 ms)Closed at 1732422342140 2024-11-24T04:25:42,145 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34457 is added to blk_1073741857_1034 (size=95) 2024-11-24T04:25:42,145 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44781 is added to blk_1073741857_1034 (size=95) 2024-11-24T04:25:42,145 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36069 is added to blk_1073741857_1034 (size=95) 2024-11-24T04:25:42,148 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-24T04:25:42,148 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1732422341272) 2024-11-24T04:25:42,168 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayValueCompression#testReplayEditsWrittenIntoWAL Thread=403 (was 387) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6816aaf7-45a5-053e-0edc-4c140671afba/cluster_be0b9e06-9b41-befd-1079-43744c8c01cc/data/data5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_747786212_22 at /127.0.0.1:46638 [Waiting for operation #8] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_747786212_22 at /127.0.0.1:49908 [Waiting for operation #8] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-12-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_747786212_22 at /127.0.0.1:33944 [Waiting for operation #22] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6816aaf7-45a5-053e-0edc-4c140671afba/cluster_be0b9e06-9b41-befd-1079-43744c8c01cc/data/data2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1319272049) connection to localhost/127.0.0.1:44559 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:44559 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6816aaf7-45a5-053e-0edc-4c140671afba/cluster_be0b9e06-9b41-befd-1079-43744c8c01cc/data/data6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_747786212_22 at /127.0.0.1:49934 [Waiting for operation #10] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkins.replay.wal.secondtime@localhost:44611 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:33657 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1319272049) connection to localhost/127.0.0.1:33657 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6816aaf7-45a5-053e-0edc-4c140671afba/cluster_be0b9e06-9b41-befd-1079-43744c8c01cc/data/data1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-12-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-12-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1319272049) connection to localhost/127.0.0.1:44611 from jenkins.replay.wal.secondtime java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) - Thread LEAK? -, OpenFileDescriptor=865 (was 785) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=196 (was 204), ProcessCount=11 (was 11), AvailableMemoryMB=6490 (was 6577) 2024-11-24T04:25:42,180 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayValueCompression#test2727 Thread=403, OpenFileDescriptor=865, MaxFileDescriptor=1048576, SystemLoadAverage=196, ProcessCount=11, AvailableMemoryMB=6489 2024-11-24T04:25:42,196 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T04:25:42,199 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T04:25:42,200 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-24T04:25:42,202 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-37197669, suffix=, logDir=hdfs://localhost:44611/hbase/WALs/hregion-37197669, archiveDir=hdfs://localhost:44611/hbase/oldWALs, maxLogs=32 2024-11-24T04:25:42,217 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-37197669/hregion-37197669.1732422342203, exclude list is [], retry=0 2024-11-24T04:25:42,220 DEBUG [AsyncFSWAL-14-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34457,DS-20bb5930-23a3-441d-bdb2-015c0d56b2ec,DISK] 2024-11-24T04:25:42,220 DEBUG [AsyncFSWAL-14-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44781,DS-54254ac8-0fae-43f7-ab06-53faa0e05dd7,DISK] 2024-11-24T04:25:42,221 DEBUG [AsyncFSWAL-14-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36069,DS-11c8f594-4fc2-47e4-b222-092ec50eb991,DISK] 2024-11-24T04:25:42,223 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-37197669/hregion-37197669.1732422342203 2024-11-24T04:25:42,223 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33347:33347),(127.0.0.1/127.0.0.1:35691:35691),(127.0.0.1/127.0.0.1:41009:41009)] 2024-11-24T04:25:42,224 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 62677596c6830b41daeb65d2cd9b4797, NAME => 'test2727,,1732422342197.62677596c6830b41daeb65d2cd9b4797.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='test2727', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44611/hbase 2024-11-24T04:25:42,234 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44781 is added to blk_1073741865_1042 (size=43) 2024-11-24T04:25:42,234 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34457 is added to blk_1073741865_1042 (size=43) 2024-11-24T04:25:42,235 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36069 is added to blk_1073741865_1042 (size=43) 2024-11-24T04:25:42,236 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated test2727,,1732422342197.62677596c6830b41daeb65d2cd9b4797.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T04:25:42,237 INFO [StoreOpener-62677596c6830b41daeb65d2cd9b4797-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 62677596c6830b41daeb65d2cd9b4797 2024-11-24T04:25:42,239 INFO [StoreOpener-62677596c6830b41daeb65d2cd9b4797-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 62677596c6830b41daeb65d2cd9b4797 columnFamilyName a 2024-11-24T04:25:42,239 DEBUG [StoreOpener-62677596c6830b41daeb65d2cd9b4797-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:25:42,239 INFO [StoreOpener-62677596c6830b41daeb65d2cd9b4797-1 {}] regionserver.HStore(327): Store=62677596c6830b41daeb65d2cd9b4797/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:25:42,240 INFO [StoreOpener-62677596c6830b41daeb65d2cd9b4797-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 62677596c6830b41daeb65d2cd9b4797 2024-11-24T04:25:42,241 INFO [StoreOpener-62677596c6830b41daeb65d2cd9b4797-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 62677596c6830b41daeb65d2cd9b4797 columnFamilyName b 2024-11-24T04:25:42,241 DEBUG [StoreOpener-62677596c6830b41daeb65d2cd9b4797-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:25:42,242 INFO [StoreOpener-62677596c6830b41daeb65d2cd9b4797-1 {}] regionserver.HStore(327): Store=62677596c6830b41daeb65d2cd9b4797/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:25:42,242 INFO [StoreOpener-62677596c6830b41daeb65d2cd9b4797-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 62677596c6830b41daeb65d2cd9b4797 2024-11-24T04:25:42,244 INFO [StoreOpener-62677596c6830b41daeb65d2cd9b4797-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 62677596c6830b41daeb65d2cd9b4797 columnFamilyName c 2024-11-24T04:25:42,244 DEBUG [StoreOpener-62677596c6830b41daeb65d2cd9b4797-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:25:42,244 INFO [StoreOpener-62677596c6830b41daeb65d2cd9b4797-1 {}] regionserver.HStore(327): Store=62677596c6830b41daeb65d2cd9b4797/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:25:42,244 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 62677596c6830b41daeb65d2cd9b4797 2024-11-24T04:25:42,245 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44611/hbase/data/default/test2727/62677596c6830b41daeb65d2cd9b4797 2024-11-24T04:25:42,246 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44611/hbase/data/default/test2727/62677596c6830b41daeb65d2cd9b4797 2024-11-24T04:25:42,247 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 62677596c6830b41daeb65d2cd9b4797 2024-11-24T04:25:42,247 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 62677596c6830b41daeb65d2cd9b4797 2024-11-24T04:25:42,248 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table test2727 descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-24T04:25:42,249 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 62677596c6830b41daeb65d2cd9b4797 2024-11-24T04:25:42,252 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44611/hbase/data/default/test2727/62677596c6830b41daeb65d2cd9b4797/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T04:25:42,253 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 62677596c6830b41daeb65d2cd9b4797; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65126534, jitterRate=-0.029539018869400024}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-24T04:25:42,254 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 62677596c6830b41daeb65d2cd9b4797: Writing region info on filesystem at 1732422342236Initializing all the Stores at 1732422342237 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732422342237Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732422342237Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732422342237Cleaning up temporary data from old regions at 1732422342247 (+10 ms)Region opened successfully at 1732422342254 (+7 ms) 2024-11-24T04:25:42,254 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 62677596c6830b41daeb65d2cd9b4797, disabling compactions & flushes 2024-11-24T04:25:42,254 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region test2727,,1732422342197.62677596c6830b41daeb65d2cd9b4797. 2024-11-24T04:25:42,254 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on test2727,,1732422342197.62677596c6830b41daeb65d2cd9b4797. 2024-11-24T04:25:42,254 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on test2727,,1732422342197.62677596c6830b41daeb65d2cd9b4797. after waiting 0 ms 2024-11-24T04:25:42,254 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region test2727,,1732422342197.62677596c6830b41daeb65d2cd9b4797. 2024-11-24T04:25:42,255 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed test2727,,1732422342197.62677596c6830b41daeb65d2cd9b4797. 2024-11-24T04:25:42,255 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 62677596c6830b41daeb65d2cd9b4797: Waiting for close lock at 1732422342254Disabling compacts and flushes for region at 1732422342254Disabling writes for close at 1732422342254Writing region close event to WAL at 1732422342255 (+1 ms)Closed at 1732422342255 2024-11-24T04:25:42,258 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34457 is added to blk_1073741864_1041 (size=95) 2024-11-24T04:25:42,258 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36069 is added to blk_1073741864_1041 (size=95) 2024-11-24T04:25:42,258 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44781 is added to blk_1073741864_1041 (size=95) 2024-11-24T04:25:42,262 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-24T04:25:42,262 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-37197669:(num 1732422342203) 2024-11-24T04:25:42,262 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-24T04:25:42,265 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:44611/hbase/WALs/test2727-manual,16010,1732422342196, archiveDir=hdfs://localhost:44611/hbase/oldWALs, maxLogs=32 2024-11-24T04:25:42,282 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/test2727-manual,16010,1732422342196/wal.1732422342265, exclude list is [], retry=0 2024-11-24T04:25:42,285 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44781,DS-54254ac8-0fae-43f7-ab06-53faa0e05dd7,DISK] 2024-11-24T04:25:42,285 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34457,DS-20bb5930-23a3-441d-bdb2-015c0d56b2ec,DISK] 2024-11-24T04:25:42,286 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36069,DS-11c8f594-4fc2-47e4-b222-092ec50eb991,DISK] 2024-11-24T04:25:42,288 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/test2727-manual,16010,1732422342196/wal.1732422342265 2024-11-24T04:25:42,288 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35691:35691),(127.0.0.1/127.0.0.1:33347:33347),(127.0.0.1/127.0.0.1:41009:41009)] 2024-11-24T04:25:42,458 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36069 is added to blk_1073741866_1043 (size=263359) 2024-11-24T04:25:42,458 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /hbase/WALs/test2727-manual,16010,1732422342196/wal.1732422342265 not finished, retry = 0 2024-11-24T04:25:42,458 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44781 is added to blk_1073741866_1043 (size=263359) 2024-11-24T04:25:42,459 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34457 is added to blk_1073741866_1043 (size=263359) 2024-11-24T04:25:42,580 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:44611/hbase/WALs/test2727-manual,16010,1732422342196/wal.1732422342265, size=257.2 K (263359bytes) 2024-11-24T04:25:42,580 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:44611/hbase/WALs/test2727-manual,16010,1732422342196/wal.1732422342265 2024-11-24T04:25:42,580 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:44611/hbase/WALs/test2727-manual,16010,1732422342196/wal.1732422342265 after 0ms 2024-11-24T04:25:42,584 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:44611/hbase/WALs/test2727-manual,16010,1732422342196/wal.1732422342265: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-24T04:25:42,585 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:44611/hbase/WALs/test2727-manual,16010,1732422342196/wal.1732422342265 took 6ms 2024-11-24T04:25:42,590 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal.1732422342265.temp 2024-11-24T04:25:42,591 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:44611/hbase/data/default/test2727/62677596c6830b41daeb65d2cd9b4797/recovered.edits/0000000000000000001-wal.1732422342265.temp 2024-11-24T04:25:42,657 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:44611/hbase/WALs/test2727-manual,16010,1732422342196/wal.1732422342265 so closing down 2024-11-24T04:25:42,657 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-24T04:25:42,658 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-24T04:25:42,663 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44781 is added to blk_1073741867_1044 (size=263359) 2024-11-24T04:25:42,663 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36069 is added to blk_1073741867_1044 (size=263359) 2024-11-24T04:25:42,665 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34457 is added to blk_1073741867_1044 (size=263359) 2024-11-24T04:25:42,665 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:44611/hbase/data/default/test2727/62677596c6830b41daeb65d2cd9b4797/recovered.edits/0000000000000000001-wal.1732422342265.temp (wrote 3000 edits, skipped 0 edits in 44 ms) 2024-11-24T04:25:42,667 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:44611/hbase/data/default/test2727/62677596c6830b41daeb65d2cd9b4797/recovered.edits/0000000000000000001-wal.1732422342265.temp to hdfs://localhost:44611/hbase/data/default/test2727/62677596c6830b41daeb65d2cd9b4797/recovered.edits/0000000000000003000 2024-11-24T04:25:42,667 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 3000 edits across 1 Regions in 82 ms; skipped=0; WAL=hdfs://localhost:44611/hbase/WALs/test2727-manual,16010,1732422342196/wal.1732422342265, size=257.2 K, length=263359, corrupted=false, cancelled=false 2024-11-24T04:25:42,667 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:44611/hbase/WALs/test2727-manual,16010,1732422342196/wal.1732422342265, journal: Splitting hdfs://localhost:44611/hbase/WALs/test2727-manual,16010,1732422342196/wal.1732422342265, size=257.2 K (263359bytes) at 1732422342580Creating recovered edits writer path=hdfs://localhost:44611/hbase/data/default/test2727/62677596c6830b41daeb65d2cd9b4797/recovered.edits/0000000000000000001-wal.1732422342265.temp at 1732422342592 (+12 ms)Split 1024 edits, skipped 0 edits. at 1732422342611 (+19 ms)Split 2048 edits, skipped 0 edits. at 1732422342635 (+24 ms)Finishing writing output for hdfs://localhost:44611/hbase/WALs/test2727-manual,16010,1732422342196/wal.1732422342265 so closing down at 1732422342657 (+22 ms)3 split writer threads finished at 1732422342658 (+1 ms)Closed recovered edits writer path=hdfs://localhost:44611/hbase/data/default/test2727/62677596c6830b41daeb65d2cd9b4797/recovered.edits/0000000000000000001-wal.1732422342265.temp (wrote 3000 edits, skipped 0 edits in 44 ms) at 1732422342665 (+7 ms)Rename recovered edits hdfs://localhost:44611/hbase/data/default/test2727/62677596c6830b41daeb65d2cd9b4797/recovered.edits/0000000000000000001-wal.1732422342265.temp to hdfs://localhost:44611/hbase/data/default/test2727/62677596c6830b41daeb65d2cd9b4797/recovered.edits/0000000000000003000 at 1732422342667 (+2 ms)Processed 3000 edits across 1 Regions in 82 ms; skipped=0; WAL=hdfs://localhost:44611/hbase/WALs/test2727-manual,16010,1732422342196/wal.1732422342265, size=257.2 K, length=263359, corrupted=false, cancelled=false at 1732422342667 2024-11-24T04:25:42,669 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:44611/hbase/WALs/test2727-manual,16010,1732422342196/wal.1732422342265 to hdfs://localhost:44611/hbase/oldWALs/wal.1732422342265 2024-11-24T04:25:42,670 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:44611/hbase/data/default/test2727/62677596c6830b41daeb65d2cd9b4797/recovered.edits/0000000000000003000 2024-11-24T04:25:42,670 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-24T04:25:42,672 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:44611/hbase/WALs/test2727-manual,16010,1732422342196, archiveDir=hdfs://localhost:44611/hbase/oldWALs, maxLogs=32 2024-11-24T04:25:42,688 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/test2727-manual,16010,1732422342196/wal.1732422342673, exclude list is [], retry=0 2024-11-24T04:25:42,691 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34457,DS-20bb5930-23a3-441d-bdb2-015c0d56b2ec,DISK] 2024-11-24T04:25:42,692 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44781,DS-54254ac8-0fae-43f7-ab06-53faa0e05dd7,DISK] 2024-11-24T04:25:42,692 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36069,DS-11c8f594-4fc2-47e4-b222-092ec50eb991,DISK] 2024-11-24T04:25:42,695 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/test2727-manual,16010,1732422342196/wal.1732422342673 2024-11-24T04:25:42,695 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33347:33347),(127.0.0.1/127.0.0.1:35691:35691),(127.0.0.1/127.0.0.1:41009:41009)] 2024-11-24T04:25:42,843 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44781 is added to blk_1073741868_1045 (size=263486) 2024-11-24T04:25:42,843 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34457 is added to blk_1073741868_1045 (size=263486) 2024-11-24T04:25:42,844 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36069 is added to blk_1073741868_1045 (size=263486) 2024-11-24T04:25:42,863 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:44611/hbase/WALs/test2727-manual,16010,1732422342196/wal.1732422342673, size=257.3 K (263486bytes) 2024-11-24T04:25:42,863 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:44611/hbase/WALs/test2727-manual,16010,1732422342196/wal.1732422342673 2024-11-24T04:25:42,863 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:44611/hbase/WALs/test2727-manual,16010,1732422342196/wal.1732422342673 after 0ms 2024-11-24T04:25:42,866 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:44611/hbase/WALs/test2727-manual,16010,1732422342196/wal.1732422342673: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-24T04:25:42,868 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:44611/hbase/WALs/test2727-manual,16010,1732422342196/wal.1732422342673 took 6ms 2024-11-24T04:25:42,872 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000003001-wal.1732422342673.temp 2024-11-24T04:25:42,874 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:44611/hbase/data/default/test2727/62677596c6830b41daeb65d2cd9b4797/recovered.edits/0000000000000003001-wal.1732422342673.temp 2024-11-24T04:25:42,929 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:44611/hbase/WALs/test2727-manual,16010,1732422342196/wal.1732422342673 so closing down 2024-11-24T04:25:42,929 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-24T04:25:42,929 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-24T04:25:42,932 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44781 is added to blk_1073741869_1046 (size=263486) 2024-11-24T04:25:42,933 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36069 is added to blk_1073741869_1046 (size=263486) 2024-11-24T04:25:42,933 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34457 is added to blk_1073741869_1046 (size=263486) 2024-11-24T04:25:42,934 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:44611/hbase/data/default/test2727/62677596c6830b41daeb65d2cd9b4797/recovered.edits/0000000000000003001-wal.1732422342673.temp (wrote 3000 edits, skipped 0 edits in 43 ms) 2024-11-24T04:25:42,936 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:44611/hbase/data/default/test2727/62677596c6830b41daeb65d2cd9b4797/recovered.edits/0000000000000003001-wal.1732422342673.temp to hdfs://localhost:44611/hbase/data/default/test2727/62677596c6830b41daeb65d2cd9b4797/recovered.edits/0000000000000006000 2024-11-24T04:25:42,936 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 3000 edits across 1 Regions in 68 ms; skipped=0; WAL=hdfs://localhost:44611/hbase/WALs/test2727-manual,16010,1732422342196/wal.1732422342673, size=257.3 K, length=263486, corrupted=false, cancelled=false 2024-11-24T04:25:42,936 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:44611/hbase/WALs/test2727-manual,16010,1732422342196/wal.1732422342673, journal: Splitting hdfs://localhost:44611/hbase/WALs/test2727-manual,16010,1732422342196/wal.1732422342673, size=257.3 K (263486bytes) at 1732422342863Creating recovered edits writer path=hdfs://localhost:44611/hbase/data/default/test2727/62677596c6830b41daeb65d2cd9b4797/recovered.edits/0000000000000003001-wal.1732422342673.temp at 1732422342874 (+11 ms)Split 1024 edits, skipped 0 edits. at 1732422342890 (+16 ms)Split 2048 edits, skipped 0 edits. at 1732422342910 (+20 ms)Finishing writing output for hdfs://localhost:44611/hbase/WALs/test2727-manual,16010,1732422342196/wal.1732422342673 so closing down at 1732422342929 (+19 ms)3 split writer threads finished at 1732422342930 (+1 ms)Closed recovered edits writer path=hdfs://localhost:44611/hbase/data/default/test2727/62677596c6830b41daeb65d2cd9b4797/recovered.edits/0000000000000003001-wal.1732422342673.temp (wrote 3000 edits, skipped 0 edits in 43 ms) at 1732422342934 (+4 ms)Rename recovered edits hdfs://localhost:44611/hbase/data/default/test2727/62677596c6830b41daeb65d2cd9b4797/recovered.edits/0000000000000003001-wal.1732422342673.temp to hdfs://localhost:44611/hbase/data/default/test2727/62677596c6830b41daeb65d2cd9b4797/recovered.edits/0000000000000006000 at 1732422342936 (+2 ms)Processed 3000 edits across 1 Regions in 68 ms; skipped=0; WAL=hdfs://localhost:44611/hbase/WALs/test2727-manual,16010,1732422342196/wal.1732422342673, size=257.3 K, length=263486, corrupted=false, cancelled=false at 1732422342936 2024-11-24T04:25:42,938 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:44611/hbase/WALs/test2727-manual,16010,1732422342196/wal.1732422342673 to hdfs://localhost:44611/hbase/oldWALs/wal.1732422342673 2024-11-24T04:25:42,940 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:44611/hbase/data/default/test2727/62677596c6830b41daeb65d2cd9b4797/recovered.edits/0000000000000006000 2024-11-24T04:25:42,940 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-24T04:25:42,942 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:44611/hbase/WALs/test2727-manual,16010,1732422342196, archiveDir=hdfs://localhost:44611/hbase/oldWALs, maxLogs=32 2024-11-24T04:25:42,966 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/test2727-manual,16010,1732422342196/wal.1732422342943, exclude list is [], retry=0 2024-11-24T04:25:42,969 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44781,DS-54254ac8-0fae-43f7-ab06-53faa0e05dd7,DISK] 2024-11-24T04:25:42,970 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34457,DS-20bb5930-23a3-441d-bdb2-015c0d56b2ec,DISK] 2024-11-24T04:25:42,970 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36069,DS-11c8f594-4fc2-47e4-b222-092ec50eb991,DISK] 2024-11-24T04:25:42,973 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/test2727-manual,16010,1732422342196/wal.1732422342943 2024-11-24T04:25:42,974 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35691:35691),(127.0.0.1/127.0.0.1:33347:33347),(127.0.0.1/127.0.0.1:41009:41009)] 2024-11-24T04:25:42,974 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 62677596c6830b41daeb65d2cd9b4797, NAME => 'test2727,,1732422342197.62677596c6830b41daeb65d2cd9b4797.', STARTKEY => '', ENDKEY => ''} 2024-11-24T04:25:42,974 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated test2727,,1732422342197.62677596c6830b41daeb65d2cd9b4797.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T04:25:42,974 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 62677596c6830b41daeb65d2cd9b4797 2024-11-24T04:25:42,974 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 62677596c6830b41daeb65d2cd9b4797 2024-11-24T04:25:42,976 INFO [StoreOpener-62677596c6830b41daeb65d2cd9b4797-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 62677596c6830b41daeb65d2cd9b4797 2024-11-24T04:25:42,977 INFO [StoreOpener-62677596c6830b41daeb65d2cd9b4797-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 62677596c6830b41daeb65d2cd9b4797 columnFamilyName a 2024-11-24T04:25:42,977 DEBUG [StoreOpener-62677596c6830b41daeb65d2cd9b4797-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:25:42,978 INFO [StoreOpener-62677596c6830b41daeb65d2cd9b4797-1 {}] regionserver.HStore(327): Store=62677596c6830b41daeb65d2cd9b4797/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:25:42,978 INFO [StoreOpener-62677596c6830b41daeb65d2cd9b4797-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 62677596c6830b41daeb65d2cd9b4797 2024-11-24T04:25:42,979 INFO [StoreOpener-62677596c6830b41daeb65d2cd9b4797-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 62677596c6830b41daeb65d2cd9b4797 columnFamilyName b 2024-11-24T04:25:42,979 DEBUG [StoreOpener-62677596c6830b41daeb65d2cd9b4797-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:25:42,980 INFO [StoreOpener-62677596c6830b41daeb65d2cd9b4797-1 {}] regionserver.HStore(327): Store=62677596c6830b41daeb65d2cd9b4797/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:25:42,980 INFO [StoreOpener-62677596c6830b41daeb65d2cd9b4797-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 62677596c6830b41daeb65d2cd9b4797 2024-11-24T04:25:42,981 INFO [StoreOpener-62677596c6830b41daeb65d2cd9b4797-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 62677596c6830b41daeb65d2cd9b4797 columnFamilyName c 2024-11-24T04:25:42,981 DEBUG [StoreOpener-62677596c6830b41daeb65d2cd9b4797-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:25:42,982 INFO [StoreOpener-62677596c6830b41daeb65d2cd9b4797-1 {}] regionserver.HStore(327): Store=62677596c6830b41daeb65d2cd9b4797/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:25:42,982 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 62677596c6830b41daeb65d2cd9b4797 2024-11-24T04:25:42,982 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44611/hbase/data/default/test2727/62677596c6830b41daeb65d2cd9b4797 2024-11-24T04:25:42,985 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 2 recovered edits file(s) under hdfs://localhost:44611/hbase/data/default/test2727/62677596c6830b41daeb65d2cd9b4797 2024-11-24T04:25:42,985 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:44611/hbase/data/default/test2727/62677596c6830b41daeb65d2cd9b4797/recovered.edits/0000000000000003000 2024-11-24T04:25:42,988 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:44611/hbase/data/default/test2727/62677596c6830b41daeb65d2cd9b4797/recovered.edits/0000000000000003000: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-24T04:25:43,037 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 3000, skipped 0, firstSequenceIdInLog=1, maxSequenceIdInLog=3000, path=hdfs://localhost:44611/hbase/data/default/test2727/62677596c6830b41daeb65d2cd9b4797/recovered.edits/0000000000000003000 2024-11-24T04:25:43,038 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:44611/hbase/data/default/test2727/62677596c6830b41daeb65d2cd9b4797/recovered.edits/0000000000000006000 2024-11-24T04:25:43,041 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:44611/hbase/data/default/test2727/62677596c6830b41daeb65d2cd9b4797/recovered.edits/0000000000000006000: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-24T04:25:43,104 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 3000, skipped 0, firstSequenceIdInLog=3001, maxSequenceIdInLog=6000, path=hdfs://localhost:44611/hbase/data/default/test2727/62677596c6830b41daeb65d2cd9b4797/recovered.edits/0000000000000006000 2024-11-24T04:25:43,105 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 62677596c6830b41daeb65d2cd9b4797 3/3 column families, dataSize=215.51 KB heapSize=657 KB 2024-11-24T04:25:43,131 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44611/hbase/data/default/test2727/62677596c6830b41daeb65d2cd9b4797/.tmp/a/4d8dab4194be471db151cf16cf8876b3 is 41, key is test2727/a:100/1732422342702/Put/seqid=0 2024-11-24T04:25:43,139 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44781 is added to blk_1073741871_1048 (size=84227) 2024-11-24T04:25:43,139 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36069 is added to blk_1073741871_1048 (size=84227) 2024-11-24T04:25:43,139 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34457 is added to blk_1073741871_1048 (size=84227) 2024-11-24T04:25:43,141 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=71.84 KB at sequenceid=6000 (bloomFilter=true), to=hdfs://localhost:44611/hbase/data/default/test2727/62677596c6830b41daeb65d2cd9b4797/.tmp/a/4d8dab4194be471db151cf16cf8876b3 2024-11-24T04:25:43,173 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44611/hbase/data/default/test2727/62677596c6830b41daeb65d2cd9b4797/.tmp/b/ca1847660fd14e9095e193e595df11f0 is 41, key is test2727/b:100/1732422342752/Put/seqid=0 2024-11-24T04:25:43,177 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36069 is added to blk_1073741872_1049 (size=84609) 2024-11-24T04:25:43,177 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34457 is added to blk_1073741872_1049 (size=84609) 2024-11-24T04:25:43,178 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44781 is added to blk_1073741872_1049 (size=84609) 2024-11-24T04:25:43,178 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=71.84 KB at sequenceid=6000 (bloomFilter=true), to=hdfs://localhost:44611/hbase/data/default/test2727/62677596c6830b41daeb65d2cd9b4797/.tmp/b/ca1847660fd14e9095e193e595df11f0 2024-11-24T04:25:43,216 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44611/hbase/data/default/test2727/62677596c6830b41daeb65d2cd9b4797/.tmp/c/0898b365770f454dbce908e5729945d3 is 41, key is test2727/c:100/1732422342799/Put/seqid=0 2024-11-24T04:25:43,223 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36069 is added to blk_1073741873_1050 (size=84609) 2024-11-24T04:25:43,223 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34457 is added to blk_1073741873_1050 (size=84609) 2024-11-24T04:25:43,224 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44781 is added to blk_1073741873_1050 (size=84609) 2024-11-24T04:25:43,310 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-24T04:25:43,310 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-24T04:25:43,313 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-24T04:25:43,313 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-24T04:25:43,313 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-24T04:25:43,313 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-24T04:25:43,314 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenIntoWAL 2024-11-24T04:25:43,314 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenIntoWAL Metrics about Tables on a single HBase RegionServer 2024-11-24T04:25:43,624 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=71.84 KB at sequenceid=6000 (bloomFilter=true), to=hdfs://localhost:44611/hbase/data/default/test2727/62677596c6830b41daeb65d2cd9b4797/.tmp/c/0898b365770f454dbce908e5729945d3 2024-11-24T04:25:43,630 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44611/hbase/data/default/test2727/62677596c6830b41daeb65d2cd9b4797/.tmp/a/4d8dab4194be471db151cf16cf8876b3 as hdfs://localhost:44611/hbase/data/default/test2727/62677596c6830b41daeb65d2cd9b4797/a/4d8dab4194be471db151cf16cf8876b3 2024-11-24T04:25:43,636 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44611/hbase/data/default/test2727/62677596c6830b41daeb65d2cd9b4797/a/4d8dab4194be471db151cf16cf8876b3, entries=2000, sequenceid=6000, filesize=82.3 K 2024-11-24T04:25:43,637 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44611/hbase/data/default/test2727/62677596c6830b41daeb65d2cd9b4797/.tmp/b/ca1847660fd14e9095e193e595df11f0 as hdfs://localhost:44611/hbase/data/default/test2727/62677596c6830b41daeb65d2cd9b4797/b/ca1847660fd14e9095e193e595df11f0 2024-11-24T04:25:43,644 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44611/hbase/data/default/test2727/62677596c6830b41daeb65d2cd9b4797/b/ca1847660fd14e9095e193e595df11f0, entries=2000, sequenceid=6000, filesize=82.6 K 2024-11-24T04:25:43,645 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44611/hbase/data/default/test2727/62677596c6830b41daeb65d2cd9b4797/.tmp/c/0898b365770f454dbce908e5729945d3 as hdfs://localhost:44611/hbase/data/default/test2727/62677596c6830b41daeb65d2cd9b4797/c/0898b365770f454dbce908e5729945d3 2024-11-24T04:25:43,651 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44611/hbase/data/default/test2727/62677596c6830b41daeb65d2cd9b4797/c/0898b365770f454dbce908e5729945d3, entries=2000, sequenceid=6000, filesize=82.6 K 2024-11-24T04:25:43,651 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~215.51 KB/220680, heapSize ~656.95 KB/672720, currentSize=0 B/0 for 62677596c6830b41daeb65d2cd9b4797 in 547ms, sequenceid=6000, compaction requested=false; wal=null 2024-11-24T04:25:43,652 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:44611/hbase/data/default/test2727/62677596c6830b41daeb65d2cd9b4797/recovered.edits/0000000000000003000 2024-11-24T04:25:43,653 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:44611/hbase/data/default/test2727/62677596c6830b41daeb65d2cd9b4797/recovered.edits/0000000000000006000 2024-11-24T04:25:43,654 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 62677596c6830b41daeb65d2cd9b4797 2024-11-24T04:25:43,654 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 62677596c6830b41daeb65d2cd9b4797 2024-11-24T04:25:43,654 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table test2727 descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-24T04:25:43,656 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 62677596c6830b41daeb65d2cd9b4797 2024-11-24T04:25:43,658 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44611/hbase/data/default/test2727/62677596c6830b41daeb65d2cd9b4797/recovered.edits/6000.seqid, newMaxSeqId=6000, maxSeqId=1 2024-11-24T04:25:43,659 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 62677596c6830b41daeb65d2cd9b4797; next sequenceid=6001; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64281398, jitterRate=-0.04213252663612366}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-24T04:25:43,661 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 62677596c6830b41daeb65d2cd9b4797: Writing region info on filesystem at 1732422342974Initializing all the Stores at 1732422342975 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732422342975Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732422342976 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732422342976Obtaining lock to block concurrent updates at 1732422343105 (+129 ms)Preparing flush snapshotting stores in 62677596c6830b41daeb65d2cd9b4797 at 1732422343105Finished memstore snapshotting test2727,,1732422342197.62677596c6830b41daeb65d2cd9b4797., syncing WAL and waiting on mvcc, flushsize=dataSize=220680, getHeapSize=672720, getOffHeapSize=0, getCellsCount=6000 at 1732422343105Flushing stores of test2727,,1732422342197.62677596c6830b41daeb65d2cd9b4797. at 1732422343105Flushing 62677596c6830b41daeb65d2cd9b4797/a: creating writer at 1732422343105Flushing 62677596c6830b41daeb65d2cd9b4797/a: appending metadata at 1732422343130 (+25 ms)Flushing 62677596c6830b41daeb65d2cd9b4797/a: closing flushed file at 1732422343130Flushing 62677596c6830b41daeb65d2cd9b4797/b: creating writer at 1732422343148 (+18 ms)Flushing 62677596c6830b41daeb65d2cd9b4797/b: appending metadata at 1732422343170 (+22 ms)Flushing 62677596c6830b41daeb65d2cd9b4797/b: closing flushed file at 1732422343170Flushing 62677596c6830b41daeb65d2cd9b4797/c: creating writer at 1732422343185 (+15 ms)Flushing 62677596c6830b41daeb65d2cd9b4797/c: appending metadata at 1732422343214 (+29 ms)Flushing 62677596c6830b41daeb65d2cd9b4797/c: closing flushed file at 1732422343214Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@46d1b802: reopening flushed file at 1732422343630 (+416 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7fb43dda: reopening flushed file at 1732422343636 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4719da54: reopening flushed file at 1732422343644 (+8 ms)Finished flush of dataSize ~215.51 KB/220680, heapSize ~656.95 KB/672720, currentSize=0 B/0 for 62677596c6830b41daeb65d2cd9b4797 in 547ms, sequenceid=6000, compaction requested=false; wal=null at 1732422343651 (+7 ms)Cleaning up temporary data from old regions at 1732422343654 (+3 ms)Region opened successfully at 1732422343661 (+7 ms) 2024-11-24T04:25:43,662 DEBUG [Time-limited test {}] wal.AbstractTestWALReplay(320): region.getOpenSeqNum(): 6001, wal3.id: 0 2024-11-24T04:25:43,662 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 62677596c6830b41daeb65d2cd9b4797, disabling compactions & flushes 2024-11-24T04:25:43,662 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region test2727,,1732422342197.62677596c6830b41daeb65d2cd9b4797. 2024-11-24T04:25:43,662 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on test2727,,1732422342197.62677596c6830b41daeb65d2cd9b4797. 2024-11-24T04:25:43,662 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on test2727,,1732422342197.62677596c6830b41daeb65d2cd9b4797. after waiting 0 ms 2024-11-24T04:25:43,662 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region test2727,,1732422342197.62677596c6830b41daeb65d2cd9b4797. 2024-11-24T04:25:43,663 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed test2727,,1732422342197.62677596c6830b41daeb65d2cd9b4797. 2024-11-24T04:25:43,663 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 62677596c6830b41daeb65d2cd9b4797: Waiting for close lock at 1732422343662Disabling compacts and flushes for region at 1732422343662Disabling writes for close at 1732422343662Writing region close event to WAL at 1732422343663 (+1 ms)Closed at 1732422343663 2024-11-24T04:25:43,667 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /hbase/WALs/test2727-manual,16010,1732422342196/wal.1732422342943 not finished, retry = 0 2024-11-24T04:25:43,668 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44781 is added to blk_1073741870_1047 (size=95) 2024-11-24T04:25:43,669 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36069 is added to blk_1073741870_1047 (size=95) 2024-11-24T04:25:43,669 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34457 is added to blk_1073741870_1047 (size=95) 2024-11-24T04:25:43,770 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-24T04:25:43,770 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1732422342943) 2024-11-24T04:25:43,785 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayValueCompression#test2727 Thread=405 (was 403) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1575425110_22 at /127.0.0.1:46638 [Waiting for operation #16] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-14-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1575425110_22 at /127.0.0.1:49908 [Waiting for operation #14] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1575425110_22 at /127.0.0.1:34050 [Waiting for operation #3] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-14-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-14-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=927 (was 865) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=196 (was 196), ProcessCount=11 (was 11), AvailableMemoryMB=6269 (was 6489) 2024-11-24T04:25:43,797 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayValueCompression#testSequentialEditLogSeqNum Thread=407, OpenFileDescriptor=927, MaxFileDescriptor=1048576, SystemLoadAverage=196, ProcessCount=11, AvailableMemoryMB=6268 2024-11-24T04:25:43,814 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T04:25:43,822 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:44611/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732422343814, archiveDir=hdfs://localhost:44611/hbase/oldWALs, maxLogs=32 2024-11-24T04:25:43,823 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732422343823 2024-11-24T04:25:43,831 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732422343814/wal.1732422343823 2024-11-24T04:25:43,833 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new MockWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33347:33347),(127.0.0.1/127.0.0.1:41009:41009),(127.0.0.1/127.0.0.1:35691:35691)] 2024-11-24T04:25:43,834 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 3c5497f80d9d26aafae7512ddc360727, NAME => 'testSequentialEditLogSeqNum,,1732422343815.3c5497f80d9d26aafae7512ddc360727.', STARTKEY => '', ENDKEY => ''} 2024-11-24T04:25:43,834 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testSequentialEditLogSeqNum,,1732422343815.3c5497f80d9d26aafae7512ddc360727.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T04:25:43,834 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 3c5497f80d9d26aafae7512ddc360727 2024-11-24T04:25:43,834 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 3c5497f80d9d26aafae7512ddc360727 2024-11-24T04:25:43,835 WARN [Time-limited test {}] regionserver.HRegionFileSystem(836): hdfs://localhost:44611/hbase/data/default/testSequentialEditLogSeqNum/3c5497f80d9d26aafae7512ddc360727 doesn't exist for region: 3c5497f80d9d26aafae7512ddc360727 on table testSequentialEditLogSeqNum 2024-11-24T04:25:43,836 WARN [Time-limited test {}] regionserver.HRegionFileSystem(854): .regioninfo file not found for region: 3c5497f80d9d26aafae7512ddc360727 on table testSequentialEditLogSeqNum 2024-11-24T04:25:43,844 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34457 is added to blk_1073741875_1052 (size=62) 2024-11-24T04:25:43,844 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44781 is added to blk_1073741875_1052 (size=62) 2024-11-24T04:25:43,845 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36069 is added to blk_1073741875_1052 (size=62) 2024-11-24T04:25:43,847 INFO [StoreOpener-3c5497f80d9d26aafae7512ddc360727-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 3c5497f80d9d26aafae7512ddc360727 2024-11-24T04:25:43,848 INFO [StoreOpener-3c5497f80d9d26aafae7512ddc360727-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3c5497f80d9d26aafae7512ddc360727 columnFamilyName a 2024-11-24T04:25:43,848 DEBUG [StoreOpener-3c5497f80d9d26aafae7512ddc360727-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:25:43,849 INFO [StoreOpener-3c5497f80d9d26aafae7512ddc360727-1 {}] regionserver.HStore(327): Store=3c5497f80d9d26aafae7512ddc360727/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:25:43,849 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 3c5497f80d9d26aafae7512ddc360727 2024-11-24T04:25:43,850 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44611/hbase/data/default/testSequentialEditLogSeqNum/3c5497f80d9d26aafae7512ddc360727 2024-11-24T04:25:43,850 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44611/hbase/data/default/testSequentialEditLogSeqNum/3c5497f80d9d26aafae7512ddc360727 2024-11-24T04:25:43,851 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 3c5497f80d9d26aafae7512ddc360727 2024-11-24T04:25:43,851 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 3c5497f80d9d26aafae7512ddc360727 2024-11-24T04:25:43,852 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 3c5497f80d9d26aafae7512ddc360727 2024-11-24T04:25:43,854 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44611/hbase/data/default/testSequentialEditLogSeqNum/3c5497f80d9d26aafae7512ddc360727/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T04:25:43,855 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 3c5497f80d9d26aafae7512ddc360727; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67378014, jitterRate=0.004010647535324097}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-24T04:25:43,856 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 3c5497f80d9d26aafae7512ddc360727: Writing region info on filesystem at 1732422343835Initializing all the Stores at 1732422343846 (+11 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732422343846Cleaning up temporary data from old regions at 1732422343851 (+5 ms)Region opened successfully at 1732422343856 (+5 ms) 2024-11-24T04:25:43,869 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 3c5497f80d9d26aafae7512ddc360727 1/1 column families, dataSize=770 B heapSize=1.73 KB 2024-11-24T04:25:43,892 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44611/hbase/data/default/testSequentialEditLogSeqNum/3c5497f80d9d26aafae7512ddc360727/.tmp/a/3b8622f1b233494d887d7f1a6a5b3f30 is 81, key is testSequentialEditLogSeqNum/a:x0/1732422343856/Put/seqid=0 2024-11-24T04:25:43,935 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36069 is added to blk_1073741876_1053 (size=5833) 2024-11-24T04:25:43,935 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34457 is added to blk_1073741876_1053 (size=5833) 2024-11-24T04:25:43,936 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44781 is added to blk_1073741876_1053 (size=5833) 2024-11-24T04:25:44,335 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=770 B at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:44611/hbase/data/default/testSequentialEditLogSeqNum/3c5497f80d9d26aafae7512ddc360727/.tmp/a/3b8622f1b233494d887d7f1a6a5b3f30 2024-11-24T04:25:44,342 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44611/hbase/data/default/testSequentialEditLogSeqNum/3c5497f80d9d26aafae7512ddc360727/.tmp/a/3b8622f1b233494d887d7f1a6a5b3f30 as hdfs://localhost:44611/hbase/data/default/testSequentialEditLogSeqNum/3c5497f80d9d26aafae7512ddc360727/a/3b8622f1b233494d887d7f1a6a5b3f30 2024-11-24T04:25:44,348 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44611/hbase/data/default/testSequentialEditLogSeqNum/3c5497f80d9d26aafae7512ddc360727/a/3b8622f1b233494d887d7f1a6a5b3f30, entries=10, sequenceid=13, filesize=5.7 K 2024-11-24T04:25:44,350 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~770 B/770, heapSize ~1.72 KB/1760, currentSize=0 B/0 for 3c5497f80d9d26aafae7512ddc360727 in 481ms, sequenceid=13, compaction requested=false 2024-11-24T04:25:44,350 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 3c5497f80d9d26aafae7512ddc360727: 2024-11-24T04:25:44,355 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T04:25:44,356 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T04:25:44,356 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T04:25:44,356 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T04:25:44,356 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T04:25:44,358 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44781 is added to blk_1073741874_1051 (size=1844) 2024-11-24T04:25:44,358 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36069 is added to blk_1073741874_1051 (size=1844) 2024-11-24T04:25:44,359 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34457 is added to blk_1073741874_1051 (size=1844) 2024-11-24T04:25:44,375 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:44611/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732422343814/wal.1732422343823, size=1.8 K (1844bytes) 2024-11-24T04:25:44,375 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:44611/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732422343814/wal.1732422343823 2024-11-24T04:25:44,376 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:44611/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732422343814/wal.1732422343823 after 1ms 2024-11-24T04:25:44,378 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:44611/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732422343814/wal.1732422343823: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-24T04:25:44,378 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:44611/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732422343814/wal.1732422343823 took 3ms 2024-11-24T04:25:44,381 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:44611/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732422343814/wal.1732422343823 so closing down 2024-11-24T04:25:44,381 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-24T04:25:44,382 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1732422343823.temp 2024-11-24T04:25:44,383 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:44611/hbase/data/default/testSequentialEditLogSeqNum/3c5497f80d9d26aafae7512ddc360727/recovered.edits/0000000000000000003-wal.1732422343823.temp 2024-11-24T04:25:44,384 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-24T04:25:44,390 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36069 is added to blk_1073741877_1054 (size=1477) 2024-11-24T04:25:44,390 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34457 is added to blk_1073741877_1054 (size=1477) 2024-11-24T04:25:44,390 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44781 is added to blk_1073741877_1054 (size=1477) 2024-11-24T04:25:44,391 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:44611/hbase/data/default/testSequentialEditLogSeqNum/3c5497f80d9d26aafae7512ddc360727/recovered.edits/0000000000000000003-wal.1732422343823.temp (wrote 15 edits, skipped 0 edits in 0 ms) 2024-11-24T04:25:44,393 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:44611/hbase/data/default/testSequentialEditLogSeqNum/3c5497f80d9d26aafae7512ddc360727/recovered.edits/0000000000000000003-wal.1732422343823.temp to hdfs://localhost:44611/hbase/data/default/testSequentialEditLogSeqNum/3c5497f80d9d26aafae7512ddc360727/recovered.edits/0000000000000000020 2024-11-24T04:25:44,393 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 17 edits across 1 Regions in 14 ms; skipped=2; WAL=hdfs://localhost:44611/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732422343814/wal.1732422343823, size=1.8 K, length=1844, corrupted=false, cancelled=false 2024-11-24T04:25:44,393 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:44611/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732422343814/wal.1732422343823, journal: Splitting hdfs://localhost:44611/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732422343814/wal.1732422343823, size=1.8 K (1844bytes) at 1732422344375Finishing writing output for hdfs://localhost:44611/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732422343814/wal.1732422343823 so closing down at 1732422344381 (+6 ms)Creating recovered edits writer path=hdfs://localhost:44611/hbase/data/default/testSequentialEditLogSeqNum/3c5497f80d9d26aafae7512ddc360727/recovered.edits/0000000000000000003-wal.1732422343823.temp at 1732422344383 (+2 ms)3 split writer threads finished at 1732422344384 (+1 ms)Closed recovered edits writer path=hdfs://localhost:44611/hbase/data/default/testSequentialEditLogSeqNum/3c5497f80d9d26aafae7512ddc360727/recovered.edits/0000000000000000003-wal.1732422343823.temp (wrote 15 edits, skipped 0 edits in 0 ms) at 1732422344391 (+7 ms)Rename recovered edits hdfs://localhost:44611/hbase/data/default/testSequentialEditLogSeqNum/3c5497f80d9d26aafae7512ddc360727/recovered.edits/0000000000000000003-wal.1732422343823.temp to hdfs://localhost:44611/hbase/data/default/testSequentialEditLogSeqNum/3c5497f80d9d26aafae7512ddc360727/recovered.edits/0000000000000000020 at 1732422344393 (+2 ms)Processed 17 edits across 1 Regions in 14 ms; skipped=2; WAL=hdfs://localhost:44611/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732422343814/wal.1732422343823, size=1.8 K, length=1844, corrupted=false, cancelled=false at 1732422344393 2024-11-24T04:25:44,409 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayValueCompression#testSequentialEditLogSeqNum Thread=412 (was 407) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1575425110_22 at /127.0.0.1:46638 [Waiting for operation #18] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: sync.0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1575425110_22 at /127.0.0.1:34050 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: sync.3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) - Thread LEAK? -, OpenFileDescriptor=963 (was 927) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=196 (was 196), ProcessCount=11 (was 11), AvailableMemoryMB=6200 (was 6268) 2024-11-24T04:25:44,424 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayValueCompression#testRegionMadeOfBulkLoadedFilesOnly Thread=412, OpenFileDescriptor=963, MaxFileDescriptor=1048576, SystemLoadAverage=196, ProcessCount=11, AvailableMemoryMB=6197 2024-11-24T04:25:44,448 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T04:25:44,451 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T04:25:44,469 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-24T04:25:44,472 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-99227153, suffix=, logDir=hdfs://localhost:44611/hbase/WALs/hregion-99227153, archiveDir=hdfs://localhost:44611/hbase/oldWALs, maxLogs=32 2024-11-24T04:25:44,492 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-99227153/hregion-99227153.1732422344473, exclude list is [], retry=0 2024-11-24T04:25:44,496 DEBUG [AsyncFSWAL-17-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36069,DS-11c8f594-4fc2-47e4-b222-092ec50eb991,DISK] 2024-11-24T04:25:44,496 DEBUG [AsyncFSWAL-17-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34457,DS-20bb5930-23a3-441d-bdb2-015c0d56b2ec,DISK] 2024-11-24T04:25:44,496 DEBUG [AsyncFSWAL-17-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44781,DS-54254ac8-0fae-43f7-ab06-53faa0e05dd7,DISK] 2024-11-24T04:25:44,499 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-99227153/hregion-99227153.1732422344473 2024-11-24T04:25:44,499 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41009:41009),(127.0.0.1/127.0.0.1:33347:33347),(127.0.0.1/127.0.0.1:35691:35691)] 2024-11-24T04:25:44,499 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 76ed0f2fd0cf2ef731a1d9c7d078ece6, NAME => 'testRegionMadeOfBulkLoadedFilesOnly,,1732422344449.76ed0f2fd0cf2ef731a1d9c7d078ece6.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testRegionMadeOfBulkLoadedFilesOnly', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44611/hbase 2024-11-24T04:25:44,509 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36069 is added to blk_1073741879_1056 (size=70) 2024-11-24T04:25:44,509 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34457 is added to blk_1073741879_1056 (size=70) 2024-11-24T04:25:44,510 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44781 is added to blk_1073741879_1056 (size=70) 2024-11-24T04:25:44,510 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testRegionMadeOfBulkLoadedFilesOnly,,1732422344449.76ed0f2fd0cf2ef731a1d9c7d078ece6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T04:25:44,512 INFO [StoreOpener-76ed0f2fd0cf2ef731a1d9c7d078ece6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 76ed0f2fd0cf2ef731a1d9c7d078ece6 2024-11-24T04:25:44,513 INFO [StoreOpener-76ed0f2fd0cf2ef731a1d9c7d078ece6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 76ed0f2fd0cf2ef731a1d9c7d078ece6 columnFamilyName a 2024-11-24T04:25:44,513 DEBUG [StoreOpener-76ed0f2fd0cf2ef731a1d9c7d078ece6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:25:44,514 INFO [StoreOpener-76ed0f2fd0cf2ef731a1d9c7d078ece6-1 {}] regionserver.HStore(327): Store=76ed0f2fd0cf2ef731a1d9c7d078ece6/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:25:44,514 INFO [StoreOpener-76ed0f2fd0cf2ef731a1d9c7d078ece6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 76ed0f2fd0cf2ef731a1d9c7d078ece6 2024-11-24T04:25:44,515 INFO [StoreOpener-76ed0f2fd0cf2ef731a1d9c7d078ece6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 76ed0f2fd0cf2ef731a1d9c7d078ece6 columnFamilyName b 2024-11-24T04:25:44,515 DEBUG [StoreOpener-76ed0f2fd0cf2ef731a1d9c7d078ece6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:25:44,516 INFO [StoreOpener-76ed0f2fd0cf2ef731a1d9c7d078ece6-1 {}] regionserver.HStore(327): Store=76ed0f2fd0cf2ef731a1d9c7d078ece6/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:25:44,516 INFO [StoreOpener-76ed0f2fd0cf2ef731a1d9c7d078ece6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 76ed0f2fd0cf2ef731a1d9c7d078ece6 2024-11-24T04:25:44,517 INFO [StoreOpener-76ed0f2fd0cf2ef731a1d9c7d078ece6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 76ed0f2fd0cf2ef731a1d9c7d078ece6 columnFamilyName c 2024-11-24T04:25:44,517 DEBUG [StoreOpener-76ed0f2fd0cf2ef731a1d9c7d078ece6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:25:44,518 INFO [StoreOpener-76ed0f2fd0cf2ef731a1d9c7d078ece6-1 {}] regionserver.HStore(327): Store=76ed0f2fd0cf2ef731a1d9c7d078ece6/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:25:44,518 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 76ed0f2fd0cf2ef731a1d9c7d078ece6 2024-11-24T04:25:44,519 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44611/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/76ed0f2fd0cf2ef731a1d9c7d078ece6 2024-11-24T04:25:44,519 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44611/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/76ed0f2fd0cf2ef731a1d9c7d078ece6 2024-11-24T04:25:44,520 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 76ed0f2fd0cf2ef731a1d9c7d078ece6 2024-11-24T04:25:44,520 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 76ed0f2fd0cf2ef731a1d9c7d078ece6 2024-11-24T04:25:44,521 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRegionMadeOfBulkLoadedFilesOnly descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-24T04:25:44,522 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 76ed0f2fd0cf2ef731a1d9c7d078ece6 2024-11-24T04:25:44,525 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44611/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/76ed0f2fd0cf2ef731a1d9c7d078ece6/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T04:25:44,525 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 76ed0f2fd0cf2ef731a1d9c7d078ece6; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64854132, jitterRate=-0.033598124980926514}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-24T04:25:44,526 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 76ed0f2fd0cf2ef731a1d9c7d078ece6: Writing region info on filesystem at 1732422344510Initializing all the Stores at 1732422344511 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732422344511Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732422344511Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732422344511Cleaning up temporary data from old regions at 1732422344520 (+9 ms)Region opened successfully at 1732422344526 (+6 ms) 2024-11-24T04:25:44,526 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 76ed0f2fd0cf2ef731a1d9c7d078ece6, disabling compactions & flushes 2024-11-24T04:25:44,526 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testRegionMadeOfBulkLoadedFilesOnly,,1732422344449.76ed0f2fd0cf2ef731a1d9c7d078ece6. 2024-11-24T04:25:44,526 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testRegionMadeOfBulkLoadedFilesOnly,,1732422344449.76ed0f2fd0cf2ef731a1d9c7d078ece6. 2024-11-24T04:25:44,526 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testRegionMadeOfBulkLoadedFilesOnly,,1732422344449.76ed0f2fd0cf2ef731a1d9c7d078ece6. after waiting 0 ms 2024-11-24T04:25:44,526 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testRegionMadeOfBulkLoadedFilesOnly,,1732422344449.76ed0f2fd0cf2ef731a1d9c7d078ece6. 2024-11-24T04:25:44,527 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testRegionMadeOfBulkLoadedFilesOnly,,1732422344449.76ed0f2fd0cf2ef731a1d9c7d078ece6. 2024-11-24T04:25:44,527 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 76ed0f2fd0cf2ef731a1d9c7d078ece6: Waiting for close lock at 1732422344526Disabling compacts and flushes for region at 1732422344526Disabling writes for close at 1732422344526Writing region close event to WAL at 1732422344527 (+1 ms)Closed at 1732422344527 2024-11-24T04:25:44,530 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44781 is added to blk_1073741878_1055 (size=95) 2024-11-24T04:25:44,530 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34457 is added to blk_1073741878_1055 (size=95) 2024-11-24T04:25:44,530 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36069 is added to blk_1073741878_1055 (size=95) 2024-11-24T04:25:44,532 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-24T04:25:44,533 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-99227153:(num 1732422344473) 2024-11-24T04:25:44,533 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-24T04:25:44,535 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:44611/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732422344447, archiveDir=hdfs://localhost:44611/hbase/oldWALs, maxLogs=32 2024-11-24T04:25:44,549 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732422344447/wal.1732422344535, exclude list is [], retry=0 2024-11-24T04:25:44,551 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44781,DS-54254ac8-0fae-43f7-ab06-53faa0e05dd7,DISK] 2024-11-24T04:25:44,552 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34457,DS-20bb5930-23a3-441d-bdb2-015c0d56b2ec,DISK] 2024-11-24T04:25:44,552 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36069,DS-11c8f594-4fc2-47e4-b222-092ec50eb991,DISK] 2024-11-24T04:25:44,554 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732422344447/wal.1732422344535 2024-11-24T04:25:44,554 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35691:35691),(127.0.0.1/127.0.0.1:33347:33347),(127.0.0.1/127.0.0.1:41009:41009)] 2024-11-24T04:25:44,554 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 76ed0f2fd0cf2ef731a1d9c7d078ece6, NAME => 'testRegionMadeOfBulkLoadedFilesOnly,,1732422344449.76ed0f2fd0cf2ef731a1d9c7d078ece6.', STARTKEY => '', ENDKEY => ''} 2024-11-24T04:25:44,555 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testRegionMadeOfBulkLoadedFilesOnly,,1732422344449.76ed0f2fd0cf2ef731a1d9c7d078ece6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T04:25:44,555 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 76ed0f2fd0cf2ef731a1d9c7d078ece6 2024-11-24T04:25:44,555 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 76ed0f2fd0cf2ef731a1d9c7d078ece6 2024-11-24T04:25:44,556 INFO [StoreOpener-76ed0f2fd0cf2ef731a1d9c7d078ece6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 76ed0f2fd0cf2ef731a1d9c7d078ece6 2024-11-24T04:25:44,557 INFO [StoreOpener-76ed0f2fd0cf2ef731a1d9c7d078ece6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 76ed0f2fd0cf2ef731a1d9c7d078ece6 columnFamilyName a 2024-11-24T04:25:44,557 DEBUG [StoreOpener-76ed0f2fd0cf2ef731a1d9c7d078ece6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:25:44,558 INFO [StoreOpener-76ed0f2fd0cf2ef731a1d9c7d078ece6-1 {}] regionserver.HStore(327): Store=76ed0f2fd0cf2ef731a1d9c7d078ece6/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:25:44,558 INFO [StoreOpener-76ed0f2fd0cf2ef731a1d9c7d078ece6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 76ed0f2fd0cf2ef731a1d9c7d078ece6 2024-11-24T04:25:44,559 INFO [StoreOpener-76ed0f2fd0cf2ef731a1d9c7d078ece6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 76ed0f2fd0cf2ef731a1d9c7d078ece6 columnFamilyName b 2024-11-24T04:25:44,559 DEBUG [StoreOpener-76ed0f2fd0cf2ef731a1d9c7d078ece6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:25:44,559 INFO [StoreOpener-76ed0f2fd0cf2ef731a1d9c7d078ece6-1 {}] regionserver.HStore(327): Store=76ed0f2fd0cf2ef731a1d9c7d078ece6/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:25:44,559 INFO [StoreOpener-76ed0f2fd0cf2ef731a1d9c7d078ece6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 76ed0f2fd0cf2ef731a1d9c7d078ece6 2024-11-24T04:25:44,560 INFO [StoreOpener-76ed0f2fd0cf2ef731a1d9c7d078ece6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 76ed0f2fd0cf2ef731a1d9c7d078ece6 columnFamilyName c 2024-11-24T04:25:44,560 DEBUG [StoreOpener-76ed0f2fd0cf2ef731a1d9c7d078ece6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:25:44,560 INFO [StoreOpener-76ed0f2fd0cf2ef731a1d9c7d078ece6-1 {}] regionserver.HStore(327): Store=76ed0f2fd0cf2ef731a1d9c7d078ece6/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:25:44,561 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 76ed0f2fd0cf2ef731a1d9c7d078ece6 2024-11-24T04:25:44,561 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44611/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/76ed0f2fd0cf2ef731a1d9c7d078ece6 2024-11-24T04:25:44,562 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44611/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/76ed0f2fd0cf2ef731a1d9c7d078ece6 2024-11-24T04:25:44,563 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 76ed0f2fd0cf2ef731a1d9c7d078ece6 2024-11-24T04:25:44,563 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 76ed0f2fd0cf2ef731a1d9c7d078ece6 2024-11-24T04:25:44,564 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRegionMadeOfBulkLoadedFilesOnly descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-24T04:25:44,565 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 76ed0f2fd0cf2ef731a1d9c7d078ece6 2024-11-24T04:25:44,566 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 76ed0f2fd0cf2ef731a1d9c7d078ece6; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74217312, jitterRate=0.10592412948608398}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-24T04:25:44,567 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 76ed0f2fd0cf2ef731a1d9c7d078ece6: Writing region info on filesystem at 1732422344555Initializing all the Stores at 1732422344556 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732422344556Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732422344556Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732422344556Cleaning up temporary data from old regions at 1732422344563 (+7 ms)Region opened successfully at 1732422344567 (+4 ms) 2024-11-24T04:25:44,571 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44611/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile is 28, key is \x0D/a:a/1732422344570/Put/seqid=0 2024-11-24T04:25:44,577 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36069 is added to blk_1073741881_1058 (size=4826) 2024-11-24T04:25:44,578 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34457 is added to blk_1073741881_1058 (size=4826) 2024-11-24T04:25:44,578 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44781 is added to blk_1073741881_1058 (size=4826) 2024-11-24T04:25:44,579 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:44611/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile for inclusion in 76ed0f2fd0cf2ef731a1d9c7d078ece6/a 2024-11-24T04:25:44,587 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first= last=z 2024-11-24T04:25:44,587 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-11-24T04:25:44,587 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 76ed0f2fd0cf2ef731a1d9c7d078ece6: 2024-11-24T04:25:44,589 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44611/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile as hdfs://localhost:44611/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/76ed0f2fd0cf2ef731a1d9c7d078ece6/a/9187cdc10ef54c64aa7735c8e331e85e_SeqId_3_ 2024-11-24T04:25:44,590 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:44611/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile into 76ed0f2fd0cf2ef731a1d9c7d078ece6/a as hdfs://localhost:44611/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/76ed0f2fd0cf2ef731a1d9c7d078ece6/a/9187cdc10ef54c64aa7735c8e331e85e_SeqId_3_ - updating store file list. 2024-11-24T04:25:44,595 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for 9187cdc10ef54c64aa7735c8e331e85e_SeqId_3_: NONE, but ROW specified in column family configuration 2024-11-24T04:25:44,596 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:44611/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/76ed0f2fd0cf2ef731a1d9c7d078ece6/a/9187cdc10ef54c64aa7735c8e331e85e_SeqId_3_ into 76ed0f2fd0cf2ef731a1d9c7d078ece6/a 2024-11-24T04:25:44,596 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:44611/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile into 76ed0f2fd0cf2ef731a1d9c7d078ece6/a (new location: hdfs://localhost:44611/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/76ed0f2fd0cf2ef731a1d9c7d078ece6/a/9187cdc10ef54c64aa7735c8e331e85e_SeqId_3_) 2024-11-24T04:25:44,638 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:44611/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732422344447/wal.1732422344535, size=0 (0bytes) 2024-11-24T04:25:44,638 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:44611/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732422344447/wal.1732422344535 might be still open, length is 0 2024-11-24T04:25:44,639 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:44611/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732422344447/wal.1732422344535 2024-11-24T04:25:44,639 WARN [IPC Server handler 3 on default port 44611 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732422344447/wal.1732422344535 has not been closed. Lease recovery is in progress. RecoveryId = 1059 for block blk_1073741880_1057 2024-11-24T04:25:44,639 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:44611/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732422344447/wal.1732422344535 after 0ms 2024-11-24T04:25:46,837 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1575425110_22 at /127.0.0.1:50126 [Receiving block BP-1091946017-172.17.0.2-1732422329789:blk_1073741880_1057] {}] datanode.DataXceiver(331): 127.0.0.1:44781:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:50126 dst: /127.0.0.1:44781 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:44781 remote=/127.0.0.1:50126]. Total timeout mills is 60000, 57766 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T04:25:46,837 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1575425110_22 at /127.0.0.1:34122 [Receiving block BP-1091946017-172.17.0.2-1732422329789:blk_1073741880_1057] {}] datanode.DataXceiver(331): 127.0.0.1:34457:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:34122 dst: /127.0.0.1:34457 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T04:25:46,838 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1575425110_22 at /127.0.0.1:46792 [Receiving block BP-1091946017-172.17.0.2-1732422329789:blk_1073741880_1057] {}] datanode.DataXceiver(331): 127.0.0.1:36069:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:46792 dst: /127.0.0.1:36069 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T04:25:46,841 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44781 is added to blk_1073741880_1059 (size=472) 2024-11-24T04:25:46,841 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34457 is added to blk_1073741880_1059 (size=472) 2024-11-24T04:25:48,640 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:44611/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732422344447/wal.1732422344535 after 4001ms 2024-11-24T04:25:48,643 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:44611/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732422344447/wal.1732422344535: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-24T04:25:48,644 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:44611/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732422344447/wal.1732422344535 took 4006ms 2024-11-24T04:25:48,646 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:44611/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732422344447/wal.1732422344535; continuing. 2024-11-24T04:25:48,646 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:44611/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732422344447/wal.1732422344535 so closing down 2024-11-24T04:25:48,646 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-24T04:25:48,647 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000005-wal.1732422344535.temp 2024-11-24T04:25:48,649 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:44611/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/76ed0f2fd0cf2ef731a1d9c7d078ece6/recovered.edits/0000000000000000005-wal.1732422344535.temp 2024-11-24T04:25:48,649 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-24T04:25:48,655 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36069 is added to blk_1073741882_1060 (size=259) 2024-11-24T04:25:48,655 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34457 is added to blk_1073741882_1060 (size=259) 2024-11-24T04:25:48,655 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44781 is added to blk_1073741882_1060 (size=259) 2024-11-24T04:25:48,655 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:44611/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/76ed0f2fd0cf2ef731a1d9c7d078ece6/recovered.edits/0000000000000000005-wal.1732422344535.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-11-24T04:25:48,657 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:44611/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/76ed0f2fd0cf2ef731a1d9c7d078ece6/recovered.edits/0000000000000000005-wal.1732422344535.temp to hdfs://localhost:44611/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/76ed0f2fd0cf2ef731a1d9c7d078ece6/recovered.edits/0000000000000000005 2024-11-24T04:25:48,657 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 2 edits across 1 Regions in 13 ms; skipped=1; WAL=hdfs://localhost:44611/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732422344447/wal.1732422344535, size=0, length=0, corrupted=false, cancelled=false 2024-11-24T04:25:48,657 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:44611/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732422344447/wal.1732422344535, journal: Splitting hdfs://localhost:44611/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732422344447/wal.1732422344535, size=0 (0bytes) at 1732422344638Finishing writing output for hdfs://localhost:44611/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732422344447/wal.1732422344535 so closing down at 1732422348646 (+4008 ms)Creating recovered edits writer path=hdfs://localhost:44611/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/76ed0f2fd0cf2ef731a1d9c7d078ece6/recovered.edits/0000000000000000005-wal.1732422344535.temp at 1732422348649 (+3 ms)3 split writer threads finished at 1732422348649Closed recovered edits writer path=hdfs://localhost:44611/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/76ed0f2fd0cf2ef731a1d9c7d078ece6/recovered.edits/0000000000000000005-wal.1732422344535.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1732422348655 (+6 ms)Rename recovered edits hdfs://localhost:44611/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/76ed0f2fd0cf2ef731a1d9c7d078ece6/recovered.edits/0000000000000000005-wal.1732422344535.temp to hdfs://localhost:44611/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/76ed0f2fd0cf2ef731a1d9c7d078ece6/recovered.edits/0000000000000000005 at 1732422348657 (+2 ms)Processed 2 edits across 1 Regions in 13 ms; skipped=1; WAL=hdfs://localhost:44611/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732422344447/wal.1732422344535, size=0, length=0, corrupted=false, cancelled=false at 1732422348657 2024-11-24T04:25:48,659 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:44611/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732422344447/wal.1732422344535 to hdfs://localhost:44611/hbase/oldWALs/wal.1732422344535 2024-11-24T04:25:48,660 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:44611/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/76ed0f2fd0cf2ef731a1d9c7d078ece6/recovered.edits/0000000000000000005 2024-11-24T04:25:48,660 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-24T04:25:48,661 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:44611/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732422344447, archiveDir=hdfs://localhost:44611/hbase/oldWALs, maxLogs=32 2024-11-24T04:25:48,675 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732422344447/wal.1732422348662, exclude list is [], retry=0 2024-11-24T04:25:48,677 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36069,DS-11c8f594-4fc2-47e4-b222-092ec50eb991,DISK] 2024-11-24T04:25:48,678 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44781,DS-54254ac8-0fae-43f7-ab06-53faa0e05dd7,DISK] 2024-11-24T04:25:48,678 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34457,DS-20bb5930-23a3-441d-bdb2-015c0d56b2ec,DISK] 2024-11-24T04:25:48,680 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732422344447/wal.1732422348662 2024-11-24T04:25:48,681 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41009:41009),(127.0.0.1/127.0.0.1:35691:35691),(127.0.0.1/127.0.0.1:33347:33347)] 2024-11-24T04:25:48,681 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 76ed0f2fd0cf2ef731a1d9c7d078ece6, NAME => 'testRegionMadeOfBulkLoadedFilesOnly,,1732422344449.76ed0f2fd0cf2ef731a1d9c7d078ece6.', STARTKEY => '', ENDKEY => ''} 2024-11-24T04:25:48,681 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testRegionMadeOfBulkLoadedFilesOnly,,1732422344449.76ed0f2fd0cf2ef731a1d9c7d078ece6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T04:25:48,681 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 76ed0f2fd0cf2ef731a1d9c7d078ece6 2024-11-24T04:25:48,681 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 76ed0f2fd0cf2ef731a1d9c7d078ece6 2024-11-24T04:25:48,682 INFO [StoreOpener-76ed0f2fd0cf2ef731a1d9c7d078ece6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 76ed0f2fd0cf2ef731a1d9c7d078ece6 2024-11-24T04:25:48,684 INFO [StoreOpener-76ed0f2fd0cf2ef731a1d9c7d078ece6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 76ed0f2fd0cf2ef731a1d9c7d078ece6 columnFamilyName a 2024-11-24T04:25:48,684 DEBUG [StoreOpener-76ed0f2fd0cf2ef731a1d9c7d078ece6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:25:48,690 DEBUG [StoreFileOpener-76ed0f2fd0cf2ef731a1d9c7d078ece6-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for 9187cdc10ef54c64aa7735c8e331e85e_SeqId_3_: NONE, but ROW specified in column family configuration 2024-11-24T04:25:48,690 DEBUG [StoreOpener-76ed0f2fd0cf2ef731a1d9c7d078ece6-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:44611/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/76ed0f2fd0cf2ef731a1d9c7d078ece6/a/9187cdc10ef54c64aa7735c8e331e85e_SeqId_3_ 2024-11-24T04:25:48,690 INFO [StoreOpener-76ed0f2fd0cf2ef731a1d9c7d078ece6-1 {}] regionserver.HStore(327): Store=76ed0f2fd0cf2ef731a1d9c7d078ece6/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:25:48,690 INFO [StoreOpener-76ed0f2fd0cf2ef731a1d9c7d078ece6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 76ed0f2fd0cf2ef731a1d9c7d078ece6 2024-11-24T04:25:48,691 INFO [StoreOpener-76ed0f2fd0cf2ef731a1d9c7d078ece6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 76ed0f2fd0cf2ef731a1d9c7d078ece6 columnFamilyName b 2024-11-24T04:25:48,691 DEBUG [StoreOpener-76ed0f2fd0cf2ef731a1d9c7d078ece6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:25:48,692 INFO [StoreOpener-76ed0f2fd0cf2ef731a1d9c7d078ece6-1 {}] regionserver.HStore(327): Store=76ed0f2fd0cf2ef731a1d9c7d078ece6/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:25:48,692 INFO [StoreOpener-76ed0f2fd0cf2ef731a1d9c7d078ece6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 76ed0f2fd0cf2ef731a1d9c7d078ece6 2024-11-24T04:25:48,693 INFO [StoreOpener-76ed0f2fd0cf2ef731a1d9c7d078ece6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 76ed0f2fd0cf2ef731a1d9c7d078ece6 columnFamilyName c 2024-11-24T04:25:48,693 DEBUG [StoreOpener-76ed0f2fd0cf2ef731a1d9c7d078ece6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:25:48,693 INFO [StoreOpener-76ed0f2fd0cf2ef731a1d9c7d078ece6-1 {}] regionserver.HStore(327): Store=76ed0f2fd0cf2ef731a1d9c7d078ece6/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:25:48,694 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 76ed0f2fd0cf2ef731a1d9c7d078ece6 2024-11-24T04:25:48,694 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44611/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/76ed0f2fd0cf2ef731a1d9c7d078ece6 2024-11-24T04:25:48,696 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:44611/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/76ed0f2fd0cf2ef731a1d9c7d078ece6 2024-11-24T04:25:48,696 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:44611/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/76ed0f2fd0cf2ef731a1d9c7d078ece6/recovered.edits/0000000000000000005 2024-11-24T04:25:48,698 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:44611/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/76ed0f2fd0cf2ef731a1d9c7d078ece6/recovered.edits/0000000000000000005: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-24T04:25:48,699 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 1, skipped 0, firstSequenceIdInLog=5, maxSequenceIdInLog=5, path=hdfs://localhost:44611/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/76ed0f2fd0cf2ef731a1d9c7d078ece6/recovered.edits/0000000000000000005 2024-11-24T04:25:48,699 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 76ed0f2fd0cf2ef731a1d9c7d078ece6 3/3 column families, dataSize=58 B heapSize=904 B 2024-11-24T04:25:48,716 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44611/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/76ed0f2fd0cf2ef731a1d9c7d078ece6/.tmp/a/50a2d4860c4c44e8a370cad49c562cdf is 62, key is testRegionMadeOfBulkLoadedFilesOnly/a:a/1732422344601/Put/seqid=0 2024-11-24T04:25:48,722 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44781 is added to blk_1073741884_1062 (size=5149) 2024-11-24T04:25:48,722 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34457 is added to blk_1073741884_1062 (size=5149) 2024-11-24T04:25:48,723 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36069 is added to blk_1073741884_1062 (size=5149) 2024-11-24T04:25:48,723 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58 B at sequenceid=5 (bloomFilter=true), to=hdfs://localhost:44611/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/76ed0f2fd0cf2ef731a1d9c7d078ece6/.tmp/a/50a2d4860c4c44e8a370cad49c562cdf 2024-11-24T04:25:48,729 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44611/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/76ed0f2fd0cf2ef731a1d9c7d078ece6/.tmp/a/50a2d4860c4c44e8a370cad49c562cdf as hdfs://localhost:44611/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/76ed0f2fd0cf2ef731a1d9c7d078ece6/a/50a2d4860c4c44e8a370cad49c562cdf 2024-11-24T04:25:48,734 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44611/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/76ed0f2fd0cf2ef731a1d9c7d078ece6/a/50a2d4860c4c44e8a370cad49c562cdf, entries=1, sequenceid=5, filesize=5.0 K 2024-11-24T04:25:48,735 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~58 B/58, heapSize ~376 B/376, currentSize=0 B/0 for 76ed0f2fd0cf2ef731a1d9c7d078ece6 in 36ms, sequenceid=5, compaction requested=false; wal=null 2024-11-24T04:25:48,735 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:44611/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/76ed0f2fd0cf2ef731a1d9c7d078ece6/recovered.edits/0000000000000000005 2024-11-24T04:25:48,736 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 76ed0f2fd0cf2ef731a1d9c7d078ece6 2024-11-24T04:25:48,736 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 76ed0f2fd0cf2ef731a1d9c7d078ece6 2024-11-24T04:25:48,737 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRegionMadeOfBulkLoadedFilesOnly descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-24T04:25:48,739 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 76ed0f2fd0cf2ef731a1d9c7d078ece6 2024-11-24T04:25:48,741 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44611/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/76ed0f2fd0cf2ef731a1d9c7d078ece6/recovered.edits/5.seqid, newMaxSeqId=5, maxSeqId=1 2024-11-24T04:25:48,742 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 76ed0f2fd0cf2ef731a1d9c7d078ece6; next sequenceid=6; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65463355, jitterRate=-0.024519994854927063}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-24T04:25:48,742 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 76ed0f2fd0cf2ef731a1d9c7d078ece6: Writing region info on filesystem at 1732422348681Initializing all the Stores at 1732422348682 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732422348682Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732422348682Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732422348682Obtaining lock to block concurrent updates at 1732422348699 (+17 ms)Preparing flush snapshotting stores in 76ed0f2fd0cf2ef731a1d9c7d078ece6 at 1732422348699Finished memstore snapshotting testRegionMadeOfBulkLoadedFilesOnly,,1732422344449.76ed0f2fd0cf2ef731a1d9c7d078ece6., syncing WAL and waiting on mvcc, flushsize=dataSize=58, getHeapSize=856, getOffHeapSize=0, getCellsCount=1 at 1732422348699Flushing stores of testRegionMadeOfBulkLoadedFilesOnly,,1732422344449.76ed0f2fd0cf2ef731a1d9c7d078ece6. at 1732422348699Flushing 76ed0f2fd0cf2ef731a1d9c7d078ece6/a: creating writer at 1732422348699Flushing 76ed0f2fd0cf2ef731a1d9c7d078ece6/a: appending metadata at 1732422348715 (+16 ms)Flushing 76ed0f2fd0cf2ef731a1d9c7d078ece6/a: closing flushed file at 1732422348715Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@372e90be: reopening flushed file at 1732422348728 (+13 ms)Finished flush of dataSize ~58 B/58, heapSize ~376 B/376, currentSize=0 B/0 for 76ed0f2fd0cf2ef731a1d9c7d078ece6 in 36ms, sequenceid=5, compaction requested=false; wal=null at 1732422348735 (+7 ms)Cleaning up temporary data from old regions at 1732422348736 (+1 ms)Region opened successfully at 1732422348742 (+6 ms) 2024-11-24T04:25:48,745 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 76ed0f2fd0cf2ef731a1d9c7d078ece6, disabling compactions & flushes 2024-11-24T04:25:48,745 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testRegionMadeOfBulkLoadedFilesOnly,,1732422344449.76ed0f2fd0cf2ef731a1d9c7d078ece6. 2024-11-24T04:25:48,745 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testRegionMadeOfBulkLoadedFilesOnly,,1732422344449.76ed0f2fd0cf2ef731a1d9c7d078ece6. 2024-11-24T04:25:48,745 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testRegionMadeOfBulkLoadedFilesOnly,,1732422344449.76ed0f2fd0cf2ef731a1d9c7d078ece6. after waiting 0 ms 2024-11-24T04:25:48,745 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testRegionMadeOfBulkLoadedFilesOnly,,1732422344449.76ed0f2fd0cf2ef731a1d9c7d078ece6. 2024-11-24T04:25:48,746 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testRegionMadeOfBulkLoadedFilesOnly,,1732422344449.76ed0f2fd0cf2ef731a1d9c7d078ece6. 2024-11-24T04:25:48,746 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 76ed0f2fd0cf2ef731a1d9c7d078ece6: Waiting for close lock at 1732422348745Disabling compacts and flushes for region at 1732422348745Disabling writes for close at 1732422348745Writing region close event to WAL at 1732422348746 (+1 ms)Closed at 1732422348746 2024-11-24T04:25:48,749 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44781 is added to blk_1073741883_1061 (size=95) 2024-11-24T04:25:48,749 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34457 is added to blk_1073741883_1061 (size=95) 2024-11-24T04:25:48,750 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36069 is added to blk_1073741883_1061 (size=95) 2024-11-24T04:25:48,752 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-24T04:25:48,752 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1732422348662) 2024-11-24T04:25:48,767 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayValueCompression#testRegionMadeOfBulkLoadedFilesOnly Thread=415 (was 412) Potentially hanging thread: AsyncFSWAL-17-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1404985250_22 at /127.0.0.1:34130 [Waiting for operation #11] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-17-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1319272049) connection to localhost/127.0.0.1:44611 from jenkinstestRegionMadeOfBulkLoadedFilesOnly java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: LeaseRenewer:jenkinstestRegionMadeOfBulkLoadedFilesOnly@localhost:44611 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1404985250_22 at /127.0.0.1:50168 [Waiting for operation #5] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-17-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1022 (was 963) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=196 (was 196), ProcessCount=11 (was 11), AvailableMemoryMB=6195 (was 6197) 2024-11-24T04:25:48,779 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayValueCompression#testReplayEditsAfterRegionMovedWithMultiCF Thread=415, OpenFileDescriptor=1022, MaxFileDescriptor=1048576, SystemLoadAverage=196, ProcessCount=11, AvailableMemoryMB=6194 2024-11-24T04:25:48,795 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T04:25:48,799 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-24T04:25:48,804 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.AsyncConnectionImpl(321): The fetched master address is fadb2fe0ba32,33969,1732422332893 2024-11-24T04:25:48,806 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@57a15091 2024-11-24T04:25:48,807 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-24T04:25:48,809 INFO [HMaster-EventLoopGroup-2-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40950, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-24T04:25:48,813 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testReplayEditsAfterRegionMovedWithMultiCF', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-24T04:25:48,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF 2024-11-24T04:25:48,822 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_PRE_OPERATION 2024-11-24T04:25:48,824 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testReplayEditsAfterRegionMovedWithMultiCF" procId is: 4 2024-11-24T04:25:48,824 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:25:48,827 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-24T04:25:48,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-24T04:25:48,837 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36069 is added to blk_1073741885_1063 (size=694) 2024-11-24T04:25:48,837 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34457 is added to blk_1073741885_1063 (size=694) 2024-11-24T04:25:48,837 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44781 is added to blk_1073741885_1063 (size=694) 2024-11-24T04:25:48,840 INFO [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => ceffd2386e7c72b432c9dfad6355a885, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1732422348811.ceffd2386e7c72b432c9dfad6355a885.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsAfterRegionMovedWithMultiCF', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44611/user/jenkins/test-data/33c38499-d2fe-b6f0-8d8b-5809759afd19 2024-11-24T04:25:48,848 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44781 is added to blk_1073741886_1064 (size=77) 2024-11-24T04:25:48,848 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34457 is added to blk_1073741886_1064 (size=77) 2024-11-24T04:25:48,849 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36069 is added to blk_1073741886_1064 (size=77) 2024-11-24T04:25:48,849 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1732422348811.ceffd2386e7c72b432c9dfad6355a885.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T04:25:48,849 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1722): Closing ceffd2386e7c72b432c9dfad6355a885, disabling compactions & flushes 2024-11-24T04:25:48,850 INFO [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1732422348811.ceffd2386e7c72b432c9dfad6355a885. 2024-11-24T04:25:48,850 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732422348811.ceffd2386e7c72b432c9dfad6355a885. 2024-11-24T04:25:48,850 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732422348811.ceffd2386e7c72b432c9dfad6355a885. after waiting 0 ms 2024-11-24T04:25:48,850 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1732422348811.ceffd2386e7c72b432c9dfad6355a885. 2024-11-24T04:25:48,850 INFO [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732422348811.ceffd2386e7c72b432c9dfad6355a885. 2024-11-24T04:25:48,850 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1676): Region close journal for ceffd2386e7c72b432c9dfad6355a885: Waiting for close lock at 1732422348849Disabling compacts and flushes for region at 1732422348849Disabling writes for close at 1732422348850 (+1 ms)Writing region close event to WAL at 1732422348850Closed at 1732422348850 2024-11-24T04:25:48,851 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_ADD_TO_META 2024-11-24T04:25:48,856 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testReplayEditsAfterRegionMovedWithMultiCF,,1732422348811.ceffd2386e7c72b432c9dfad6355a885.","families":{"info":[{"qualifier":"regioninfo","vlen":76,"tag":[],"timestamp":"1732422348851"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732422348851"}]},"ts":"1732422348851"} 2024-11-24T04:25:48,860 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-24T04:25:48,861 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-24T04:25:48,864 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testReplayEditsAfterRegionMovedWithMultiCF","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732422348861"}]},"ts":"1732422348861"} 2024-11-24T04:25:48,868 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testReplayEditsAfterRegionMovedWithMultiCF, state=ENABLING in hbase:meta 2024-11-24T04:25:48,868 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {fadb2fe0ba32=0} racks are {/default-rack=0} 2024-11-24T04:25:48,869 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-24T04:25:48,869 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-24T04:25:48,869 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-11-24T04:25:48,869 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-24T04:25:48,869 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-24T04:25:48,869 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-11-24T04:25:48,869 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-24T04:25:48,870 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-24T04:25:48,870 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-11-24T04:25:48,870 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-24T04:25:48,871 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=ceffd2386e7c72b432c9dfad6355a885, ASSIGN}] 2024-11-24T04:25:48,873 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=ceffd2386e7c72b432c9dfad6355a885, ASSIGN 2024-11-24T04:25:48,874 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=ceffd2386e7c72b432c9dfad6355a885, ASSIGN; state=OFFLINE, location=fadb2fe0ba32,43811,1732422333705; forceNewPlan=false, retain=false 2024-11-24T04:25:48,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-24T04:25:49,026 INFO [fadb2fe0ba32:33969 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-24T04:25:49,027 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=ceffd2386e7c72b432c9dfad6355a885, regionState=OPENING, regionLocation=fadb2fe0ba32,43811,1732422333705 2024-11-24T04:25:49,031 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=ceffd2386e7c72b432c9dfad6355a885, ASSIGN because future has completed 2024-11-24T04:25:49,031 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure ceffd2386e7c72b432c9dfad6355a885, server=fadb2fe0ba32,43811,1732422333705}] 2024-11-24T04:25:49,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-24T04:25:49,185 DEBUG [RSProcedureDispatcher-pool-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-24T04:25:49,187 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48703, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-24T04:25:49,191 INFO [RS_OPEN_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1732422348811.ceffd2386e7c72b432c9dfad6355a885. 2024-11-24T04:25:49,192 DEBUG [RS_OPEN_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => ceffd2386e7c72b432c9dfad6355a885, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1732422348811.ceffd2386e7c72b432c9dfad6355a885.', STARTKEY => '', ENDKEY => ''} 2024-11-24T04:25:49,192 DEBUG [RS_OPEN_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF ceffd2386e7c72b432c9dfad6355a885 2024-11-24T04:25:49,192 DEBUG [RS_OPEN_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1732422348811.ceffd2386e7c72b432c9dfad6355a885.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T04:25:49,192 DEBUG [RS_OPEN_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for ceffd2386e7c72b432c9dfad6355a885 2024-11-24T04:25:49,192 DEBUG [RS_OPEN_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for ceffd2386e7c72b432c9dfad6355a885 2024-11-24T04:25:49,194 INFO [StoreOpener-ceffd2386e7c72b432c9dfad6355a885-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region ceffd2386e7c72b432c9dfad6355a885 2024-11-24T04:25:49,195 INFO [StoreOpener-ceffd2386e7c72b432c9dfad6355a885-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ceffd2386e7c72b432c9dfad6355a885 columnFamilyName cf1 2024-11-24T04:25:49,195 DEBUG [StoreOpener-ceffd2386e7c72b432c9dfad6355a885-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:25:49,196 INFO [StoreOpener-ceffd2386e7c72b432c9dfad6355a885-1 {}] regionserver.HStore(327): Store=ceffd2386e7c72b432c9dfad6355a885/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:25:49,196 INFO [StoreOpener-ceffd2386e7c72b432c9dfad6355a885-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region ceffd2386e7c72b432c9dfad6355a885 2024-11-24T04:25:49,197 INFO [StoreOpener-ceffd2386e7c72b432c9dfad6355a885-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ceffd2386e7c72b432c9dfad6355a885 columnFamilyName cf2 2024-11-24T04:25:49,197 DEBUG [StoreOpener-ceffd2386e7c72b432c9dfad6355a885-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:25:49,198 INFO [StoreOpener-ceffd2386e7c72b432c9dfad6355a885-1 {}] regionserver.HStore(327): Store=ceffd2386e7c72b432c9dfad6355a885/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:25:49,198 DEBUG [RS_OPEN_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for ceffd2386e7c72b432c9dfad6355a885 2024-11-24T04:25:49,199 DEBUG [RS_OPEN_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44611/user/jenkins/test-data/33c38499-d2fe-b6f0-8d8b-5809759afd19/data/default/testReplayEditsAfterRegionMovedWithMultiCF/ceffd2386e7c72b432c9dfad6355a885 2024-11-24T04:25:49,199 DEBUG [RS_OPEN_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44611/user/jenkins/test-data/33c38499-d2fe-b6f0-8d8b-5809759afd19/data/default/testReplayEditsAfterRegionMovedWithMultiCF/ceffd2386e7c72b432c9dfad6355a885 2024-11-24T04:25:49,200 DEBUG [RS_OPEN_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for ceffd2386e7c72b432c9dfad6355a885 2024-11-24T04:25:49,200 DEBUG [RS_OPEN_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for ceffd2386e7c72b432c9dfad6355a885 2024-11-24T04:25:49,201 DEBUG [RS_OPEN_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-11-24T04:25:49,202 DEBUG [RS_OPEN_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for ceffd2386e7c72b432c9dfad6355a885 2024-11-24T04:25:49,204 DEBUG [RS_OPEN_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44611/user/jenkins/test-data/33c38499-d2fe-b6f0-8d8b-5809759afd19/data/default/testReplayEditsAfterRegionMovedWithMultiCF/ceffd2386e7c72b432c9dfad6355a885/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T04:25:49,205 INFO [RS_OPEN_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened ceffd2386e7c72b432c9dfad6355a885; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71610464, jitterRate=0.06707906723022461}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-11-24T04:25:49,205 DEBUG [RS_OPEN_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for ceffd2386e7c72b432c9dfad6355a885 2024-11-24T04:25:49,206 DEBUG [RS_OPEN_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for ceffd2386e7c72b432c9dfad6355a885: Running coprocessor pre-open hook at 1732422349192Writing region info on filesystem at 1732422349192Initializing all the Stores at 1732422349193 (+1 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732422349193Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732422349194 (+1 ms)Cleaning up temporary data from old regions at 1732422349200 (+6 ms)Running coprocessor post-open hooks at 1732422349205 (+5 ms)Region opened successfully at 1732422349205 2024-11-24T04:25:49,207 INFO [RS_OPEN_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1732422348811.ceffd2386e7c72b432c9dfad6355a885., pid=6, masterSystemTime=1732422349184 2024-11-24T04:25:49,210 DEBUG [RS_OPEN_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1732422348811.ceffd2386e7c72b432c9dfad6355a885. 2024-11-24T04:25:49,210 INFO [RS_OPEN_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1732422348811.ceffd2386e7c72b432c9dfad6355a885. 2024-11-24T04:25:49,211 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=ceffd2386e7c72b432c9dfad6355a885, regionState=OPEN, openSeqNum=2, regionLocation=fadb2fe0ba32,43811,1732422333705 2024-11-24T04:25:49,214 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure ceffd2386e7c72b432c9dfad6355a885, server=fadb2fe0ba32,43811,1732422333705 because future has completed 2024-11-24T04:25:49,219 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-24T04:25:49,219 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure ceffd2386e7c72b432c9dfad6355a885, server=fadb2fe0ba32,43811,1732422333705 in 185 msec 2024-11-24T04:25:49,223 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-24T04:25:49,223 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=ceffd2386e7c72b432c9dfad6355a885, ASSIGN in 348 msec 2024-11-24T04:25:49,225 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-24T04:25:49,225 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testReplayEditsAfterRegionMovedWithMultiCF","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732422349225"}]},"ts":"1732422349225"} 2024-11-24T04:25:49,228 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testReplayEditsAfterRegionMovedWithMultiCF, state=ENABLED in hbase:meta 2024-11-24T04:25:49,230 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_POST_OPERATION 2024-11-24T04:25:49,233 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF in 415 msec 2024-11-24T04:25:49,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-24T04:25:49,459 INFO [RPCClient-NioEventLoopGroup-6-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testReplayEditsAfterRegionMovedWithMultiCF completed 2024-11-24T04:25:49,459 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testReplayEditsAfterRegionMovedWithMultiCF get assigned. Timeout = 60000ms 2024-11-24T04:25:49,461 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-24T04:25:49,466 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testReplayEditsAfterRegionMovedWithMultiCF assigned to meta. Checking AM states. 2024-11-24T04:25:49,467 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-24T04:25:49,467 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testReplayEditsAfterRegionMovedWithMultiCF assigned. 2024-11-24T04:25:49,479 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1732422348811.ceffd2386e7c72b432c9dfad6355a885., hostname=fadb2fe0ba32,43811,1732422333705, seqNum=2] 2024-11-24T04:25:49,480 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T04:25:49,482 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47254, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T04:25:49,496 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] master.HMaster(2410): Client=jenkins//172.17.0.2 move hri=ceffd2386e7c72b432c9dfad6355a885, source=fadb2fe0ba32,43811,1732422333705, destination=fadb2fe0ba32,35977,1732422333601, warming up region on fadb2fe0ba32,35977,1732422333601 2024-11-24T04:25:49,498 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] master.HMaster(2414): Client=jenkins//172.17.0.2 move hri=ceffd2386e7c72b432c9dfad6355a885, source=fadb2fe0ba32,43811,1732422333705, destination=fadb2fe0ba32,35977,1732422333601, running balancer 2024-11-24T04:25:49,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=ceffd2386e7c72b432c9dfad6355a885, REOPEN/MOVE 2024-11-24T04:25:49,501 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=ceffd2386e7c72b432c9dfad6355a885, REOPEN/MOVE 2024-11-24T04:25:49,503 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=ceffd2386e7c72b432c9dfad6355a885, regionState=CLOSING, regionLocation=fadb2fe0ba32,43811,1732422333705 2024-11-24T04:25:49,504 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35977 {}] regionserver.RSRpcServices(2066): Warmup testReplayEditsAfterRegionMovedWithMultiCF,,1732422348811.ceffd2386e7c72b432c9dfad6355a885. 2024-11-24T04:25:49,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35977 {}] regionserver.HRegion(7855): Warmup {ENCODED => ceffd2386e7c72b432c9dfad6355a885, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1732422348811.ceffd2386e7c72b432c9dfad6355a885.', STARTKEY => '', ENDKEY => ''} 2024-11-24T04:25:49,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35977 {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1732422348811.ceffd2386e7c72b432c9dfad6355a885.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T04:25:49,506 INFO [StoreOpener-ceffd2386e7c72b432c9dfad6355a885-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region ceffd2386e7c72b432c9dfad6355a885 2024-11-24T04:25:49,506 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=ceffd2386e7c72b432c9dfad6355a885, REOPEN/MOVE because future has completed 2024-11-24T04:25:49,507 INFO [StoreOpener-ceffd2386e7c72b432c9dfad6355a885-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ceffd2386e7c72b432c9dfad6355a885 columnFamilyName cf1 2024-11-24T04:25:49,507 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-24T04:25:49,507 DEBUG [StoreOpener-ceffd2386e7c72b432c9dfad6355a885-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:25:49,507 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure ceffd2386e7c72b432c9dfad6355a885, server=fadb2fe0ba32,43811,1732422333705}] 2024-11-24T04:25:49,507 INFO [StoreOpener-ceffd2386e7c72b432c9dfad6355a885-1 {}] regionserver.HStore(327): Store=ceffd2386e7c72b432c9dfad6355a885/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:25:49,508 INFO [StoreOpener-ceffd2386e7c72b432c9dfad6355a885-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region ceffd2386e7c72b432c9dfad6355a885 2024-11-24T04:25:49,509 INFO [StoreOpener-ceffd2386e7c72b432c9dfad6355a885-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ceffd2386e7c72b432c9dfad6355a885 columnFamilyName cf2 2024-11-24T04:25:49,509 DEBUG [StoreOpener-ceffd2386e7c72b432c9dfad6355a885-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:25:49,510 INFO [StoreOpener-ceffd2386e7c72b432c9dfad6355a885-1 {}] regionserver.HStore(327): Store=ceffd2386e7c72b432c9dfad6355a885/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:25:49,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35977 {}] regionserver.HRegion(1722): Closing ceffd2386e7c72b432c9dfad6355a885, disabling compactions & flushes 2024-11-24T04:25:49,510 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35977 {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1732422348811.ceffd2386e7c72b432c9dfad6355a885. 2024-11-24T04:25:49,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35977 {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732422348811.ceffd2386e7c72b432c9dfad6355a885. 2024-11-24T04:25:49,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35977 {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732422348811.ceffd2386e7c72b432c9dfad6355a885. after waiting 0 ms 2024-11-24T04:25:49,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35977 {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1732422348811.ceffd2386e7c72b432c9dfad6355a885. 2024-11-24T04:25:49,511 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35977 {}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732422348811.ceffd2386e7c72b432c9dfad6355a885. 2024-11-24T04:25:49,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35977 {}] regionserver.HRegion(1676): Region close journal for ceffd2386e7c72b432c9dfad6355a885: Waiting for close lock at 1732422349510Disabling compacts and flushes for region at 1732422349510Disabling writes for close at 1732422349510Writing region close event to WAL at 1732422349511 (+1 ms)Closed at 1732422349511 2024-11-24T04:25:49,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] procedure.ProcedureSyncWait(219): waitFor pid=7 2024-11-24T04:25:49,666 INFO [RS_CLOSE_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(122): Close ceffd2386e7c72b432c9dfad6355a885 2024-11-24T04:25:49,666 DEBUG [RS_CLOSE_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-24T04:25:49,667 DEBUG [RS_CLOSE_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1722): Closing ceffd2386e7c72b432c9dfad6355a885, disabling compactions & flushes 2024-11-24T04:25:49,667 INFO [RS_CLOSE_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1732422348811.ceffd2386e7c72b432c9dfad6355a885. 2024-11-24T04:25:49,667 DEBUG [RS_CLOSE_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732422348811.ceffd2386e7c72b432c9dfad6355a885. 2024-11-24T04:25:49,667 DEBUG [RS_CLOSE_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732422348811.ceffd2386e7c72b432c9dfad6355a885. after waiting 0 ms 2024-11-24T04:25:49,667 DEBUG [RS_CLOSE_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1732422348811.ceffd2386e7c72b432c9dfad6355a885. 2024-11-24T04:25:49,667 INFO [RS_CLOSE_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(2902): Flushing ceffd2386e7c72b432c9dfad6355a885 2/2 column families, dataSize=31 B heapSize=616 B 2024-11-24T04:25:49,685 DEBUG [RS_CLOSE_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44611/user/jenkins/test-data/33c38499-d2fe-b6f0-8d8b-5809759afd19/data/default/testReplayEditsAfterRegionMovedWithMultiCF/ceffd2386e7c72b432c9dfad6355a885/.tmp/cf1/c935a5a531c44d4fb6ae1b38d1879052 is 35, key is r1/cf1:q/1732422349483/Put/seqid=0 2024-11-24T04:25:49,694 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44781 is added to blk_1073741887_1065 (size=4783) 2024-11-24T04:25:49,695 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34457 is added to blk_1073741887_1065 (size=4783) 2024-11-24T04:25:49,695 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36069 is added to blk_1073741887_1065 (size=4783) 2024-11-24T04:25:49,695 INFO [RS_CLOSE_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31 B at sequenceid=5 (bloomFilter=false), to=hdfs://localhost:44611/user/jenkins/test-data/33c38499-d2fe-b6f0-8d8b-5809759afd19/data/default/testReplayEditsAfterRegionMovedWithMultiCF/ceffd2386e7c72b432c9dfad6355a885/.tmp/cf1/c935a5a531c44d4fb6ae1b38d1879052 2024-11-24T04:25:49,703 DEBUG [RS_CLOSE_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44611/user/jenkins/test-data/33c38499-d2fe-b6f0-8d8b-5809759afd19/data/default/testReplayEditsAfterRegionMovedWithMultiCF/ceffd2386e7c72b432c9dfad6355a885/.tmp/cf1/c935a5a531c44d4fb6ae1b38d1879052 as hdfs://localhost:44611/user/jenkins/test-data/33c38499-d2fe-b6f0-8d8b-5809759afd19/data/default/testReplayEditsAfterRegionMovedWithMultiCF/ceffd2386e7c72b432c9dfad6355a885/cf1/c935a5a531c44d4fb6ae1b38d1879052 2024-11-24T04:25:49,709 INFO [RS_CLOSE_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44611/user/jenkins/test-data/33c38499-d2fe-b6f0-8d8b-5809759afd19/data/default/testReplayEditsAfterRegionMovedWithMultiCF/ceffd2386e7c72b432c9dfad6355a885/cf1/c935a5a531c44d4fb6ae1b38d1879052, entries=1, sequenceid=5, filesize=4.7 K 2024-11-24T04:25:49,711 INFO [RS_CLOSE_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~31 B/31, heapSize ~344 B/344, currentSize=0 B/0 for ceffd2386e7c72b432c9dfad6355a885 in 44ms, sequenceid=5, compaction requested=false 2024-11-24T04:25:49,711 DEBUG [RS_CLOSE_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testReplayEditsAfterRegionMovedWithMultiCF' 2024-11-24T04:25:49,716 DEBUG [RS_CLOSE_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44611/user/jenkins/test-data/33c38499-d2fe-b6f0-8d8b-5809759afd19/data/default/testReplayEditsAfterRegionMovedWithMultiCF/ceffd2386e7c72b432c9dfad6355a885/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-24T04:25:49,719 INFO [RS_CLOSE_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732422348811.ceffd2386e7c72b432c9dfad6355a885. 2024-11-24T04:25:49,719 DEBUG [RS_CLOSE_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1676): Region close journal for ceffd2386e7c72b432c9dfad6355a885: Waiting for close lock at 1732422349667Running coprocessor pre-close hooks at 1732422349667Disabling compacts and flushes for region at 1732422349667Disabling writes for close at 1732422349667Obtaining lock to block concurrent updates at 1732422349667Preparing flush snapshotting stores in ceffd2386e7c72b432c9dfad6355a885 at 1732422349667Finished memstore snapshotting testReplayEditsAfterRegionMovedWithMultiCF,,1732422348811.ceffd2386e7c72b432c9dfad6355a885., syncing WAL and waiting on mvcc, flushsize=dataSize=31, getHeapSize=584, getOffHeapSize=0, getCellsCount=1 at 1732422349667Flushing stores of testReplayEditsAfterRegionMovedWithMultiCF,,1732422348811.ceffd2386e7c72b432c9dfad6355a885. at 1732422349668 (+1 ms)Flushing ceffd2386e7c72b432c9dfad6355a885/cf1: creating writer at 1732422349668Flushing ceffd2386e7c72b432c9dfad6355a885/cf1: appending metadata at 1732422349684 (+16 ms)Flushing ceffd2386e7c72b432c9dfad6355a885/cf1: closing flushed file at 1732422349684Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@63512fed: reopening flushed file at 1732422349702 (+18 ms)Finished flush of dataSize ~31 B/31, heapSize ~344 B/344, currentSize=0 B/0 for ceffd2386e7c72b432c9dfad6355a885 in 44ms, sequenceid=5, compaction requested=false at 1732422349711 (+9 ms)Writing region close event to WAL at 1732422349712 (+1 ms)Running coprocessor post-close hooks at 1732422349717 (+5 ms)Closed at 1732422349719 (+2 ms) 2024-11-24T04:25:49,720 INFO [RS_CLOSE_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionServer(3302): Adding ceffd2386e7c72b432c9dfad6355a885 move to fadb2fe0ba32,35977,1732422333601 record at close sequenceid=5 2024-11-24T04:25:49,723 INFO [RS_CLOSE_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(157): Closed ceffd2386e7c72b432c9dfad6355a885 2024-11-24T04:25:49,724 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=ceffd2386e7c72b432c9dfad6355a885, regionState=CLOSED 2024-11-24T04:25:49,727 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure ceffd2386e7c72b432c9dfad6355a885, server=fadb2fe0ba32,43811,1732422333705 because future has completed 2024-11-24T04:25:49,731 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-24T04:25:49,732 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; CloseRegionProcedure ceffd2386e7c72b432c9dfad6355a885, server=fadb2fe0ba32,43811,1732422333705 in 221 msec 2024-11-24T04:25:49,732 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=ceffd2386e7c72b432c9dfad6355a885, REOPEN/MOVE; state=CLOSED, location=fadb2fe0ba32,35977,1732422333601; forceNewPlan=false, retain=false 2024-11-24T04:25:49,883 INFO [fadb2fe0ba32:33969 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-24T04:25:49,883 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=ceffd2386e7c72b432c9dfad6355a885, regionState=OPENING, regionLocation=fadb2fe0ba32,35977,1732422333601 2024-11-24T04:25:49,886 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=ceffd2386e7c72b432c9dfad6355a885, REOPEN/MOVE because future has completed 2024-11-24T04:25:49,887 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure ceffd2386e7c72b432c9dfad6355a885, server=fadb2fe0ba32,35977,1732422333601}] 2024-11-24T04:25:50,044 INFO [RS_OPEN_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1732422348811.ceffd2386e7c72b432c9dfad6355a885. 2024-11-24T04:25:50,045 DEBUG [RS_OPEN_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => ceffd2386e7c72b432c9dfad6355a885, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1732422348811.ceffd2386e7c72b432c9dfad6355a885.', STARTKEY => '', ENDKEY => ''} 2024-11-24T04:25:50,045 DEBUG [RS_OPEN_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF ceffd2386e7c72b432c9dfad6355a885 2024-11-24T04:25:50,045 DEBUG [RS_OPEN_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1732422348811.ceffd2386e7c72b432c9dfad6355a885.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T04:25:50,045 DEBUG [RS_OPEN_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for ceffd2386e7c72b432c9dfad6355a885 2024-11-24T04:25:50,045 DEBUG [RS_OPEN_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for ceffd2386e7c72b432c9dfad6355a885 2024-11-24T04:25:50,048 INFO [StoreOpener-ceffd2386e7c72b432c9dfad6355a885-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region ceffd2386e7c72b432c9dfad6355a885 2024-11-24T04:25:50,050 INFO [StoreOpener-ceffd2386e7c72b432c9dfad6355a885-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ceffd2386e7c72b432c9dfad6355a885 columnFamilyName cf1 2024-11-24T04:25:50,050 DEBUG [StoreOpener-ceffd2386e7c72b432c9dfad6355a885-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:25:50,058 DEBUG [StoreOpener-ceffd2386e7c72b432c9dfad6355a885-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:44611/user/jenkins/test-data/33c38499-d2fe-b6f0-8d8b-5809759afd19/data/default/testReplayEditsAfterRegionMovedWithMultiCF/ceffd2386e7c72b432c9dfad6355a885/cf1/c935a5a531c44d4fb6ae1b38d1879052 2024-11-24T04:25:50,058 INFO [StoreOpener-ceffd2386e7c72b432c9dfad6355a885-1 {}] regionserver.HStore(327): Store=ceffd2386e7c72b432c9dfad6355a885/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:25:50,058 INFO [StoreOpener-ceffd2386e7c72b432c9dfad6355a885-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region ceffd2386e7c72b432c9dfad6355a885 2024-11-24T04:25:50,059 INFO [StoreOpener-ceffd2386e7c72b432c9dfad6355a885-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ceffd2386e7c72b432c9dfad6355a885 columnFamilyName cf2 2024-11-24T04:25:50,059 DEBUG [StoreOpener-ceffd2386e7c72b432c9dfad6355a885-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:25:50,060 INFO [StoreOpener-ceffd2386e7c72b432c9dfad6355a885-1 {}] regionserver.HStore(327): Store=ceffd2386e7c72b432c9dfad6355a885/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:25:50,060 DEBUG [RS_OPEN_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for ceffd2386e7c72b432c9dfad6355a885 2024-11-24T04:25:50,061 DEBUG [RS_OPEN_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44611/user/jenkins/test-data/33c38499-d2fe-b6f0-8d8b-5809759afd19/data/default/testReplayEditsAfterRegionMovedWithMultiCF/ceffd2386e7c72b432c9dfad6355a885 2024-11-24T04:25:50,062 DEBUG [RS_OPEN_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44611/user/jenkins/test-data/33c38499-d2fe-b6f0-8d8b-5809759afd19/data/default/testReplayEditsAfterRegionMovedWithMultiCF/ceffd2386e7c72b432c9dfad6355a885 2024-11-24T04:25:50,062 DEBUG [RS_OPEN_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for ceffd2386e7c72b432c9dfad6355a885 2024-11-24T04:25:50,062 DEBUG [RS_OPEN_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for ceffd2386e7c72b432c9dfad6355a885 2024-11-24T04:25:50,063 DEBUG [RS_OPEN_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-11-24T04:25:50,065 DEBUG [RS_OPEN_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for ceffd2386e7c72b432c9dfad6355a885 2024-11-24T04:25:50,065 INFO [RS_OPEN_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1114): Opened ceffd2386e7c72b432c9dfad6355a885; next sequenceid=9; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63225412, jitterRate=-0.05786794424057007}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-11-24T04:25:50,066 DEBUG [RS_OPEN_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for ceffd2386e7c72b432c9dfad6355a885 2024-11-24T04:25:50,066 DEBUG [RS_OPEN_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for ceffd2386e7c72b432c9dfad6355a885: Running coprocessor pre-open hook at 1732422350046Writing region info on filesystem at 1732422350046Initializing all the Stores at 1732422350047 (+1 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732422350048 (+1 ms)Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732422350048Cleaning up temporary data from old regions at 1732422350062 (+14 ms)Running coprocessor post-open hooks at 1732422350066 (+4 ms)Region opened successfully at 1732422350066 2024-11-24T04:25:50,068 INFO [RS_OPEN_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1732422348811.ceffd2386e7c72b432c9dfad6355a885., pid=9, masterSystemTime=1732422350039 2024-11-24T04:25:50,070 DEBUG [RS_OPEN_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1732422348811.ceffd2386e7c72b432c9dfad6355a885. 2024-11-24T04:25:50,070 INFO [RS_OPEN_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1732422348811.ceffd2386e7c72b432c9dfad6355a885. 2024-11-24T04:25:50,071 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=ceffd2386e7c72b432c9dfad6355a885, regionState=OPEN, openSeqNum=9, regionLocation=fadb2fe0ba32,35977,1732422333601 2024-11-24T04:25:50,073 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure ceffd2386e7c72b432c9dfad6355a885, server=fadb2fe0ba32,35977,1732422333601 because future has completed 2024-11-24T04:25:50,078 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=7 2024-11-24T04:25:50,078 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=7, state=SUCCESS, hasLock=false; OpenRegionProcedure ceffd2386e7c72b432c9dfad6355a885, server=fadb2fe0ba32,35977,1732422333601 in 188 msec 2024-11-24T04:25:50,080 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=ceffd2386e7c72b432c9dfad6355a885, REOPEN/MOVE in 580 msec 2024-11-24T04:25:50,108 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-24T04:25:50,110 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46572, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-24T04:25:50,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43811 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 139 connection: 172.17.0.2:47254 deadline: 1732422410114, exception=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=fadb2fe0ba32 port=35977 startCode=1732422333601. As of locationSeqNum=5. 2024-11-24T04:25:50,121 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732422348811.ceffd2386e7c72b432c9dfad6355a885., hostname=fadb2fe0ba32,43811,1732422333705, seqNum=2 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1732422348811.ceffd2386e7c72b432c9dfad6355a885., hostname=fadb2fe0ba32,43811,1732422333705, seqNum=2, error=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=fadb2fe0ba32 port=35977 startCode=1732422333601. As of locationSeqNum=5. 2024-11-24T04:25:50,121 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732422348811.ceffd2386e7c72b432c9dfad6355a885., hostname=fadb2fe0ba32,43811,1732422333705, seqNum=2 is org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=fadb2fe0ba32 port=35977 startCode=1732422333601. As of locationSeqNum=5. 2024-11-24T04:25:50,122 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(84): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732422348811.ceffd2386e7c72b432c9dfad6355a885., hostname=fadb2fe0ba32,43811,1732422333705, seqNum=2 with the new location region=testReplayEditsAfterRegionMovedWithMultiCF,,1732422348811.ceffd2386e7c72b432c9dfad6355a885., hostname=fadb2fe0ba32,35977,1732422333601, seqNum=5 constructed by org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=fadb2fe0ba32 port=35977 startCode=1732422333601. As of locationSeqNum=5. 2024-11-24T04:25:50,238 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing ceffd2386e7c72b432c9dfad6355a885 2/2 column families, dataSize=50 B heapSize=720 B 2024-11-24T04:25:50,257 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44611/user/jenkins/test-data/33c38499-d2fe-b6f0-8d8b-5809759afd19/data/default/testReplayEditsAfterRegionMovedWithMultiCF/ceffd2386e7c72b432c9dfad6355a885/.tmp/cf1/aec9b2b895ec4eea81aaeedfbd223a46 is 29, key is r1/cf1:/1732422350229/DeleteFamily/seqid=0 2024-11-24T04:25:50,264 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44781 is added to blk_1073741888_1066 (size=4906) 2024-11-24T04:25:50,264 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36069 is added to blk_1073741888_1066 (size=4906) 2024-11-24T04:25:50,264 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34457 is added to blk_1073741888_1066 (size=4906) 2024-11-24T04:25:50,265 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=25 B at sequenceid=12 (bloomFilter=false), to=hdfs://localhost:44611/user/jenkins/test-data/33c38499-d2fe-b6f0-8d8b-5809759afd19/data/default/testReplayEditsAfterRegionMovedWithMultiCF/ceffd2386e7c72b432c9dfad6355a885/.tmp/cf1/aec9b2b895ec4eea81aaeedfbd223a46 2024-11-24T04:25:50,271 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for aec9b2b895ec4eea81aaeedfbd223a46 2024-11-24T04:25:50,286 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44611/user/jenkins/test-data/33c38499-d2fe-b6f0-8d8b-5809759afd19/data/default/testReplayEditsAfterRegionMovedWithMultiCF/ceffd2386e7c72b432c9dfad6355a885/.tmp/cf2/02ff88f97d71415985d2c999aab28388 is 29, key is r1/cf2:/1732422350229/DeleteFamily/seqid=0 2024-11-24T04:25:50,293 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44781 is added to blk_1073741889_1067 (size=4906) 2024-11-24T04:25:50,294 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34457 is added to blk_1073741889_1067 (size=4906) 2024-11-24T04:25:50,294 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36069 is added to blk_1073741889_1067 (size=4906) 2024-11-24T04:25:50,294 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=25 B at sequenceid=12 (bloomFilter=false), to=hdfs://localhost:44611/user/jenkins/test-data/33c38499-d2fe-b6f0-8d8b-5809759afd19/data/default/testReplayEditsAfterRegionMovedWithMultiCF/ceffd2386e7c72b432c9dfad6355a885/.tmp/cf2/02ff88f97d71415985d2c999aab28388 2024-11-24T04:25:50,301 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 02ff88f97d71415985d2c999aab28388 2024-11-24T04:25:50,302 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44611/user/jenkins/test-data/33c38499-d2fe-b6f0-8d8b-5809759afd19/data/default/testReplayEditsAfterRegionMovedWithMultiCF/ceffd2386e7c72b432c9dfad6355a885/.tmp/cf1/aec9b2b895ec4eea81aaeedfbd223a46 as hdfs://localhost:44611/user/jenkins/test-data/33c38499-d2fe-b6f0-8d8b-5809759afd19/data/default/testReplayEditsAfterRegionMovedWithMultiCF/ceffd2386e7c72b432c9dfad6355a885/cf1/aec9b2b895ec4eea81aaeedfbd223a46 2024-11-24T04:25:50,309 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for aec9b2b895ec4eea81aaeedfbd223a46 2024-11-24T04:25:50,309 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44611/user/jenkins/test-data/33c38499-d2fe-b6f0-8d8b-5809759afd19/data/default/testReplayEditsAfterRegionMovedWithMultiCF/ceffd2386e7c72b432c9dfad6355a885/cf1/aec9b2b895ec4eea81aaeedfbd223a46, entries=1, sequenceid=12, filesize=4.8 K 2024-11-24T04:25:50,310 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44611/user/jenkins/test-data/33c38499-d2fe-b6f0-8d8b-5809759afd19/data/default/testReplayEditsAfterRegionMovedWithMultiCF/ceffd2386e7c72b432c9dfad6355a885/.tmp/cf2/02ff88f97d71415985d2c999aab28388 as hdfs://localhost:44611/user/jenkins/test-data/33c38499-d2fe-b6f0-8d8b-5809759afd19/data/default/testReplayEditsAfterRegionMovedWithMultiCF/ceffd2386e7c72b432c9dfad6355a885/cf2/02ff88f97d71415985d2c999aab28388 2024-11-24T04:25:50,316 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 02ff88f97d71415985d2c999aab28388 2024-11-24T04:25:50,316 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44611/user/jenkins/test-data/33c38499-d2fe-b6f0-8d8b-5809759afd19/data/default/testReplayEditsAfterRegionMovedWithMultiCF/ceffd2386e7c72b432c9dfad6355a885/cf2/02ff88f97d71415985d2c999aab28388, entries=1, sequenceid=12, filesize=4.8 K 2024-11-24T04:25:50,318 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~50 B/50, heapSize ~688 B/688, currentSize=0 B/0 for ceffd2386e7c72b432c9dfad6355a885 in 80ms, sequenceid=12, compaction requested=false 2024-11-24T04:25:50,318 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for ceffd2386e7c72b432c9dfad6355a885: 2024-11-24T04:25:50,321 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-24T04:25:50,322 DEBUG [Time-limited test {}] regionserver.HStore(1541): ceffd2386e7c72b432c9dfad6355a885/cf1 is initiating major compaction (all files) 2024-11-24T04:25:50,322 INFO [Time-limited test {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-24T04:25:50,322 INFO [Time-limited test {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T04:25:50,323 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of ceffd2386e7c72b432c9dfad6355a885/cf1 in testReplayEditsAfterRegionMovedWithMultiCF,,1732422348811.ceffd2386e7c72b432c9dfad6355a885. 2024-11-24T04:25:50,323 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44611/user/jenkins/test-data/33c38499-d2fe-b6f0-8d8b-5809759afd19/data/default/testReplayEditsAfterRegionMovedWithMultiCF/ceffd2386e7c72b432c9dfad6355a885/cf1/c935a5a531c44d4fb6ae1b38d1879052, hdfs://localhost:44611/user/jenkins/test-data/33c38499-d2fe-b6f0-8d8b-5809759afd19/data/default/testReplayEditsAfterRegionMovedWithMultiCF/ceffd2386e7c72b432c9dfad6355a885/cf1/aec9b2b895ec4eea81aaeedfbd223a46] into tmpdir=hdfs://localhost:44611/user/jenkins/test-data/33c38499-d2fe-b6f0-8d8b-5809759afd19/data/default/testReplayEditsAfterRegionMovedWithMultiCF/ceffd2386e7c72b432c9dfad6355a885/.tmp, totalSize=9.5 K 2024-11-24T04:25:50,324 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting c935a5a531c44d4fb6ae1b38d1879052, keycount=1, bloomtype=NONE, size=4.7 K, encoding=NONE, compression=NONE, seqNum=5, earliestPutTs=1732422349483 2024-11-24T04:25:50,325 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting aec9b2b895ec4eea81aaeedfbd223a46, keycount=1, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=9223372036854775807 2024-11-24T04:25:50,337 INFO [Time-limited test {}] throttle.PressureAwareThroughputController(145): ceffd2386e7c72b432c9dfad6355a885#cf1#compaction#16 average throughput is NaN MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T04:25:50,344 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44781 is added to blk_1073741890_1068 (size=4626) 2024-11-24T04:25:50,345 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36069 is added to blk_1073741890_1068 (size=4626) 2024-11-24T04:25:50,345 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34457 is added to blk_1073741890_1068 (size=4626) 2024-11-24T04:25:50,353 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44611/user/jenkins/test-data/33c38499-d2fe-b6f0-8d8b-5809759afd19/data/default/testReplayEditsAfterRegionMovedWithMultiCF/ceffd2386e7c72b432c9dfad6355a885/.tmp/cf1/8e879a9777d543179208ddd4216bbc74 as hdfs://localhost:44611/user/jenkins/test-data/33c38499-d2fe-b6f0-8d8b-5809759afd19/data/default/testReplayEditsAfterRegionMovedWithMultiCF/ceffd2386e7c72b432c9dfad6355a885/cf1/8e879a9777d543179208ddd4216bbc74 2024-11-24T04:25:50,373 INFO [Time-limited test {}] regionserver.HStore(1337): Completed major compaction of 2 (all) file(s) in ceffd2386e7c72b432c9dfad6355a885/cf1 of ceffd2386e7c72b432c9dfad6355a885 into 8e879a9777d543179208ddd4216bbc74(size=4.5 K), total size for store is 4.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T04:25:50,373 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for ceffd2386e7c72b432c9dfad6355a885: 2024-11-24T04:25:50,373 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 1 store files, 0 compacting, 1 eligible, 16 blocking 2024-11-24T04:25:50,373 DEBUG [Time-limited test {}] regionserver.HStore(1541): ceffd2386e7c72b432c9dfad6355a885/cf2 is initiating major compaction (all files) 2024-11-24T04:25:50,373 INFO [Time-limited test {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-24T04:25:50,373 INFO [Time-limited test {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T04:25:50,374 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of ceffd2386e7c72b432c9dfad6355a885/cf2 in testReplayEditsAfterRegionMovedWithMultiCF,,1732422348811.ceffd2386e7c72b432c9dfad6355a885. 2024-11-24T04:25:50,374 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44611/user/jenkins/test-data/33c38499-d2fe-b6f0-8d8b-5809759afd19/data/default/testReplayEditsAfterRegionMovedWithMultiCF/ceffd2386e7c72b432c9dfad6355a885/cf2/02ff88f97d71415985d2c999aab28388] into tmpdir=hdfs://localhost:44611/user/jenkins/test-data/33c38499-d2fe-b6f0-8d8b-5809759afd19/data/default/testReplayEditsAfterRegionMovedWithMultiCF/ceffd2386e7c72b432c9dfad6355a885/.tmp, totalSize=4.8 K 2024-11-24T04:25:50,375 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 02ff88f97d71415985d2c999aab28388, keycount=1, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=9223372036854775807 2024-11-24T04:25:50,383 INFO [Time-limited test {}] throttle.PressureAwareThroughputController(145): ceffd2386e7c72b432c9dfad6355a885#cf2#compaction#17 average throughput is NaN MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T04:25:50,395 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34457 is added to blk_1073741891_1069 (size=4592) 2024-11-24T04:25:50,395 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44781 is added to blk_1073741891_1069 (size=4592) 2024-11-24T04:25:50,400 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36069 is added to blk_1073741891_1069 (size=4592) 2024-11-24T04:25:50,402 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44611/user/jenkins/test-data/33c38499-d2fe-b6f0-8d8b-5809759afd19/data/default/testReplayEditsAfterRegionMovedWithMultiCF/ceffd2386e7c72b432c9dfad6355a885/.tmp/cf2/5f893655ca114f208aa3ea0462443f32 as hdfs://localhost:44611/user/jenkins/test-data/33c38499-d2fe-b6f0-8d8b-5809759afd19/data/default/testReplayEditsAfterRegionMovedWithMultiCF/ceffd2386e7c72b432c9dfad6355a885/cf2/5f893655ca114f208aa3ea0462443f32 2024-11-24T04:25:50,409 INFO [Time-limited test {}] regionserver.HStore(1337): Completed major compaction of 1 (all) file(s) in ceffd2386e7c72b432c9dfad6355a885/cf2 of ceffd2386e7c72b432c9dfad6355a885 into 5f893655ca114f208aa3ea0462443f32(size=4.5 K), total size for store is 4.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T04:25:50,410 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for ceffd2386e7c72b432c9dfad6355a885: 2024-11-24T04:25:50,414 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] master.HMaster(2410): Client=jenkins//172.17.0.2 move hri=ceffd2386e7c72b432c9dfad6355a885, source=fadb2fe0ba32,35977,1732422333601, destination=fadb2fe0ba32,43811,1732422333705, warming up region on fadb2fe0ba32,43811,1732422333705 2024-11-24T04:25:50,414 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] master.HMaster(2414): Client=jenkins//172.17.0.2 move hri=ceffd2386e7c72b432c9dfad6355a885, source=fadb2fe0ba32,35977,1732422333601, destination=fadb2fe0ba32,43811,1732422333705, running balancer 2024-11-24T04:25:50,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] procedure2.ProcedureExecutor(1139): Stored pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=ceffd2386e7c72b432c9dfad6355a885, REOPEN/MOVE 2024-11-24T04:25:50,416 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=ceffd2386e7c72b432c9dfad6355a885, REOPEN/MOVE 2024-11-24T04:25:50,417 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=ceffd2386e7c72b432c9dfad6355a885, regionState=CLOSING, regionLocation=fadb2fe0ba32,35977,1732422333601 2024-11-24T04:25:50,418 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43811 {}] regionserver.RSRpcServices(2066): Warmup testReplayEditsAfterRegionMovedWithMultiCF,,1732422348811.ceffd2386e7c72b432c9dfad6355a885. 2024-11-24T04:25:50,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43811 {}] regionserver.HRegion(7855): Warmup {ENCODED => ceffd2386e7c72b432c9dfad6355a885, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1732422348811.ceffd2386e7c72b432c9dfad6355a885.', STARTKEY => '', ENDKEY => ''} 2024-11-24T04:25:50,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43811 {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1732422348811.ceffd2386e7c72b432c9dfad6355a885.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T04:25:50,419 INFO [StoreOpener-ceffd2386e7c72b432c9dfad6355a885-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region ceffd2386e7c72b432c9dfad6355a885 2024-11-24T04:25:50,420 INFO [StoreOpener-ceffd2386e7c72b432c9dfad6355a885-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ceffd2386e7c72b432c9dfad6355a885 columnFamilyName cf1 2024-11-24T04:25:50,420 DEBUG [StoreOpener-ceffd2386e7c72b432c9dfad6355a885-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:25:50,420 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=ceffd2386e7c72b432c9dfad6355a885, REOPEN/MOVE because future has completed 2024-11-24T04:25:50,420 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-24T04:25:50,420 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=11, ppid=10, state=RUNNABLE, hasLock=false; CloseRegionProcedure ceffd2386e7c72b432c9dfad6355a885, server=fadb2fe0ba32,35977,1732422333601}] 2024-11-24T04:25:50,429 DEBUG [StoreOpener-ceffd2386e7c72b432c9dfad6355a885-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:44611/user/jenkins/test-data/33c38499-d2fe-b6f0-8d8b-5809759afd19/data/default/testReplayEditsAfterRegionMovedWithMultiCF/ceffd2386e7c72b432c9dfad6355a885/cf1/8e879a9777d543179208ddd4216bbc74 2024-11-24T04:25:50,433 INFO [StoreFileOpener-ceffd2386e7c72b432c9dfad6355a885-cf1-1 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for aec9b2b895ec4eea81aaeedfbd223a46 2024-11-24T04:25:50,433 DEBUG [StoreOpener-ceffd2386e7c72b432c9dfad6355a885-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:44611/user/jenkins/test-data/33c38499-d2fe-b6f0-8d8b-5809759afd19/data/default/testReplayEditsAfterRegionMovedWithMultiCF/ceffd2386e7c72b432c9dfad6355a885/cf1/aec9b2b895ec4eea81aaeedfbd223a46 2024-11-24T04:25:50,438 DEBUG [StoreOpener-ceffd2386e7c72b432c9dfad6355a885-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:44611/user/jenkins/test-data/33c38499-d2fe-b6f0-8d8b-5809759afd19/data/default/testReplayEditsAfterRegionMovedWithMultiCF/ceffd2386e7c72b432c9dfad6355a885/cf1/c935a5a531c44d4fb6ae1b38d1879052 2024-11-24T04:25:50,438 INFO [StoreOpener-ceffd2386e7c72b432c9dfad6355a885-1 {}] regionserver.HStore(327): Store=ceffd2386e7c72b432c9dfad6355a885/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:25:50,438 INFO [StoreOpener-ceffd2386e7c72b432c9dfad6355a885-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region ceffd2386e7c72b432c9dfad6355a885 2024-11-24T04:25:50,439 INFO [StoreOpener-ceffd2386e7c72b432c9dfad6355a885-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ceffd2386e7c72b432c9dfad6355a885 columnFamilyName cf2 2024-11-24T04:25:50,439 DEBUG [StoreOpener-ceffd2386e7c72b432c9dfad6355a885-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:25:50,445 INFO [StoreFileOpener-ceffd2386e7c72b432c9dfad6355a885-cf2-1 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 02ff88f97d71415985d2c999aab28388 2024-11-24T04:25:50,446 DEBUG [StoreOpener-ceffd2386e7c72b432c9dfad6355a885-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:44611/user/jenkins/test-data/33c38499-d2fe-b6f0-8d8b-5809759afd19/data/default/testReplayEditsAfterRegionMovedWithMultiCF/ceffd2386e7c72b432c9dfad6355a885/cf2/02ff88f97d71415985d2c999aab28388 2024-11-24T04:25:50,450 DEBUG [StoreOpener-ceffd2386e7c72b432c9dfad6355a885-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:44611/user/jenkins/test-data/33c38499-d2fe-b6f0-8d8b-5809759afd19/data/default/testReplayEditsAfterRegionMovedWithMultiCF/ceffd2386e7c72b432c9dfad6355a885/cf2/5f893655ca114f208aa3ea0462443f32 2024-11-24T04:25:50,450 INFO [StoreOpener-ceffd2386e7c72b432c9dfad6355a885-1 {}] regionserver.HStore(327): Store=ceffd2386e7c72b432c9dfad6355a885/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:25:50,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43811 {}] regionserver.HRegion(1722): Closing ceffd2386e7c72b432c9dfad6355a885, disabling compactions & flushes 2024-11-24T04:25:50,451 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43811 {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1732422348811.ceffd2386e7c72b432c9dfad6355a885. 2024-11-24T04:25:50,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43811 {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732422348811.ceffd2386e7c72b432c9dfad6355a885. 2024-11-24T04:25:50,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43811 {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732422348811.ceffd2386e7c72b432c9dfad6355a885. after waiting 0 ms 2024-11-24T04:25:50,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43811 {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1732422348811.ceffd2386e7c72b432c9dfad6355a885. 2024-11-24T04:25:50,452 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43811 {}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732422348811.ceffd2386e7c72b432c9dfad6355a885. 2024-11-24T04:25:50,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43811 {}] regionserver.HRegion(1676): Region close journal for ceffd2386e7c72b432c9dfad6355a885: Waiting for close lock at 1732422350450Disabling compacts and flushes for region at 1732422350450Disabling writes for close at 1732422350451 (+1 ms)Writing region close event to WAL at 1732422350452 (+1 ms)Closed at 1732422350452 2024-11-24T04:25:50,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] procedure.ProcedureSyncWait(219): waitFor pid=10 2024-11-24T04:25:50,575 INFO [RS_CLOSE_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] handler.UnassignRegionHandler(122): Close ceffd2386e7c72b432c9dfad6355a885 2024-11-24T04:25:50,575 DEBUG [RS_CLOSE_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-24T04:25:50,575 DEBUG [RS_CLOSE_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1722): Closing ceffd2386e7c72b432c9dfad6355a885, disabling compactions & flushes 2024-11-24T04:25:50,575 INFO [RS_CLOSE_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1732422348811.ceffd2386e7c72b432c9dfad6355a885. 2024-11-24T04:25:50,575 DEBUG [RS_CLOSE_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732422348811.ceffd2386e7c72b432c9dfad6355a885. 2024-11-24T04:25:50,575 DEBUG [RS_CLOSE_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732422348811.ceffd2386e7c72b432c9dfad6355a885. after waiting 0 ms 2024-11-24T04:25:50,575 DEBUG [RS_CLOSE_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1732422348811.ceffd2386e7c72b432c9dfad6355a885. 2024-11-24T04:25:50,576 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1732422348811.ceffd2386e7c72b432c9dfad6355a885.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44611/user/jenkins/test-data/33c38499-d2fe-b6f0-8d8b-5809759afd19/data/default/testReplayEditsAfterRegionMovedWithMultiCF/ceffd2386e7c72b432c9dfad6355a885/cf1/c935a5a531c44d4fb6ae1b38d1879052, hdfs://localhost:44611/user/jenkins/test-data/33c38499-d2fe-b6f0-8d8b-5809759afd19/data/default/testReplayEditsAfterRegionMovedWithMultiCF/ceffd2386e7c72b432c9dfad6355a885/cf1/aec9b2b895ec4eea81aaeedfbd223a46] to archive 2024-11-24T04:25:50,579 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1732422348811.ceffd2386e7c72b432c9dfad6355a885.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-24T04:25:50,582 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1732422348811.ceffd2386e7c72b432c9dfad6355a885.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44611/user/jenkins/test-data/33c38499-d2fe-b6f0-8d8b-5809759afd19/data/default/testReplayEditsAfterRegionMovedWithMultiCF/ceffd2386e7c72b432c9dfad6355a885/cf1/c935a5a531c44d4fb6ae1b38d1879052 to hdfs://localhost:44611/user/jenkins/test-data/33c38499-d2fe-b6f0-8d8b-5809759afd19/archive/data/default/testReplayEditsAfterRegionMovedWithMultiCF/ceffd2386e7c72b432c9dfad6355a885/cf1/c935a5a531c44d4fb6ae1b38d1879052 2024-11-24T04:25:50,584 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1732422348811.ceffd2386e7c72b432c9dfad6355a885.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44611/user/jenkins/test-data/33c38499-d2fe-b6f0-8d8b-5809759afd19/data/default/testReplayEditsAfterRegionMovedWithMultiCF/ceffd2386e7c72b432c9dfad6355a885/cf1/aec9b2b895ec4eea81aaeedfbd223a46 to hdfs://localhost:44611/user/jenkins/test-data/33c38499-d2fe-b6f0-8d8b-5809759afd19/archive/data/default/testReplayEditsAfterRegionMovedWithMultiCF/ceffd2386e7c72b432c9dfad6355a885/cf1/aec9b2b895ec4eea81aaeedfbd223a46 2024-11-24T04:25:50,597 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1732422348811.ceffd2386e7c72b432c9dfad6355a885.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44611/user/jenkins/test-data/33c38499-d2fe-b6f0-8d8b-5809759afd19/data/default/testReplayEditsAfterRegionMovedWithMultiCF/ceffd2386e7c72b432c9dfad6355a885/cf2/02ff88f97d71415985d2c999aab28388] to archive 2024-11-24T04:25:50,598 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1732422348811.ceffd2386e7c72b432c9dfad6355a885.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-24T04:25:50,600 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1732422348811.ceffd2386e7c72b432c9dfad6355a885.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44611/user/jenkins/test-data/33c38499-d2fe-b6f0-8d8b-5809759afd19/data/default/testReplayEditsAfterRegionMovedWithMultiCF/ceffd2386e7c72b432c9dfad6355a885/cf2/02ff88f97d71415985d2c999aab28388 to hdfs://localhost:44611/user/jenkins/test-data/33c38499-d2fe-b6f0-8d8b-5809759afd19/archive/data/default/testReplayEditsAfterRegionMovedWithMultiCF/ceffd2386e7c72b432c9dfad6355a885/cf2/02ff88f97d71415985d2c999aab28388 2024-11-24T04:25:50,604 DEBUG [RS_CLOSE_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44611/user/jenkins/test-data/33c38499-d2fe-b6f0-8d8b-5809759afd19/data/default/testReplayEditsAfterRegionMovedWithMultiCF/ceffd2386e7c72b432c9dfad6355a885/recovered.edits/17.seqid, newMaxSeqId=17, maxSeqId=8 2024-11-24T04:25:50,605 INFO [RS_CLOSE_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732422348811.ceffd2386e7c72b432c9dfad6355a885. 2024-11-24T04:25:50,605 DEBUG [RS_CLOSE_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1676): Region close journal for ceffd2386e7c72b432c9dfad6355a885: Waiting for close lock at 1732422350575Running coprocessor pre-close hooks at 1732422350575Disabling compacts and flushes for region at 1732422350575Disabling writes for close at 1732422350575Writing region close event to WAL at 1732422350601 (+26 ms)Running coprocessor post-close hooks at 1732422350605 (+4 ms)Closed at 1732422350605 2024-11-24T04:25:50,605 INFO [RS_CLOSE_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegionServer(3302): Adding ceffd2386e7c72b432c9dfad6355a885 move to fadb2fe0ba32,43811,1732422333705 record at close sequenceid=12 2024-11-24T04:25:50,608 INFO [RS_CLOSE_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] handler.UnassignRegionHandler(157): Closed ceffd2386e7c72b432c9dfad6355a885 2024-11-24T04:25:50,609 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=ceffd2386e7c72b432c9dfad6355a885, regionState=CLOSED 2024-11-24T04:25:50,611 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=10, state=RUNNABLE, hasLock=false; CloseRegionProcedure ceffd2386e7c72b432c9dfad6355a885, server=fadb2fe0ba32,35977,1732422333601 because future has completed 2024-11-24T04:25:50,615 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=11, resume processing ppid=10 2024-11-24T04:25:50,615 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=10, state=SUCCESS, hasLock=false; CloseRegionProcedure ceffd2386e7c72b432c9dfad6355a885, server=fadb2fe0ba32,35977,1732422333601 in 192 msec 2024-11-24T04:25:50,616 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=ceffd2386e7c72b432c9dfad6355a885, REOPEN/MOVE; state=CLOSED, location=fadb2fe0ba32,43811,1732422333705; forceNewPlan=false, retain=false 2024-11-24T04:25:50,766 INFO [fadb2fe0ba32:33969 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-24T04:25:50,767 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=ceffd2386e7c72b432c9dfad6355a885, regionState=OPENING, regionLocation=fadb2fe0ba32,43811,1732422333705 2024-11-24T04:25:50,770 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=ceffd2386e7c72b432c9dfad6355a885, REOPEN/MOVE because future has completed 2024-11-24T04:25:50,770 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure ceffd2386e7c72b432c9dfad6355a885, server=fadb2fe0ba32,43811,1732422333705}] 2024-11-24T04:25:50,927 INFO [RS_OPEN_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1732422348811.ceffd2386e7c72b432c9dfad6355a885. 2024-11-24T04:25:50,927 DEBUG [RS_OPEN_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7752): Opening region: {ENCODED => ceffd2386e7c72b432c9dfad6355a885, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1732422348811.ceffd2386e7c72b432c9dfad6355a885.', STARTKEY => '', ENDKEY => ''} 2024-11-24T04:25:50,928 DEBUG [RS_OPEN_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF ceffd2386e7c72b432c9dfad6355a885 2024-11-24T04:25:50,928 DEBUG [RS_OPEN_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1732422348811.ceffd2386e7c72b432c9dfad6355a885.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T04:25:50,928 DEBUG [RS_OPEN_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7794): checking encryption for ceffd2386e7c72b432c9dfad6355a885 2024-11-24T04:25:50,928 DEBUG [RS_OPEN_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7797): checking classloading for ceffd2386e7c72b432c9dfad6355a885 2024-11-24T04:25:50,929 INFO [StoreOpener-ceffd2386e7c72b432c9dfad6355a885-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region ceffd2386e7c72b432c9dfad6355a885 2024-11-24T04:25:50,930 INFO [StoreOpener-ceffd2386e7c72b432c9dfad6355a885-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ceffd2386e7c72b432c9dfad6355a885 columnFamilyName cf1 2024-11-24T04:25:50,930 DEBUG [StoreOpener-ceffd2386e7c72b432c9dfad6355a885-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:25:50,937 DEBUG [StoreOpener-ceffd2386e7c72b432c9dfad6355a885-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:44611/user/jenkins/test-data/33c38499-d2fe-b6f0-8d8b-5809759afd19/data/default/testReplayEditsAfterRegionMovedWithMultiCF/ceffd2386e7c72b432c9dfad6355a885/cf1/8e879a9777d543179208ddd4216bbc74 2024-11-24T04:25:50,937 INFO [StoreOpener-ceffd2386e7c72b432c9dfad6355a885-1 {}] regionserver.HStore(327): Store=ceffd2386e7c72b432c9dfad6355a885/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:25:50,937 INFO [StoreOpener-ceffd2386e7c72b432c9dfad6355a885-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region ceffd2386e7c72b432c9dfad6355a885 2024-11-24T04:25:50,938 INFO [StoreOpener-ceffd2386e7c72b432c9dfad6355a885-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ceffd2386e7c72b432c9dfad6355a885 columnFamilyName cf2 2024-11-24T04:25:50,938 DEBUG [StoreOpener-ceffd2386e7c72b432c9dfad6355a885-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:25:50,945 DEBUG [StoreOpener-ceffd2386e7c72b432c9dfad6355a885-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:44611/user/jenkins/test-data/33c38499-d2fe-b6f0-8d8b-5809759afd19/data/default/testReplayEditsAfterRegionMovedWithMultiCF/ceffd2386e7c72b432c9dfad6355a885/cf2/5f893655ca114f208aa3ea0462443f32 2024-11-24T04:25:50,946 INFO [StoreOpener-ceffd2386e7c72b432c9dfad6355a885-1 {}] regionserver.HStore(327): Store=ceffd2386e7c72b432c9dfad6355a885/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:25:50,946 DEBUG [RS_OPEN_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1038): replaying wal for ceffd2386e7c72b432c9dfad6355a885 2024-11-24T04:25:50,947 DEBUG [RS_OPEN_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44611/user/jenkins/test-data/33c38499-d2fe-b6f0-8d8b-5809759afd19/data/default/testReplayEditsAfterRegionMovedWithMultiCF/ceffd2386e7c72b432c9dfad6355a885 2024-11-24T04:25:50,950 DEBUG [RS_OPEN_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44611/user/jenkins/test-data/33c38499-d2fe-b6f0-8d8b-5809759afd19/data/default/testReplayEditsAfterRegionMovedWithMultiCF/ceffd2386e7c72b432c9dfad6355a885 2024-11-24T04:25:50,951 DEBUG [RS_OPEN_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1048): stopping wal replay for ceffd2386e7c72b432c9dfad6355a885 2024-11-24T04:25:50,951 DEBUG [RS_OPEN_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1060): Cleaning up temporary data for ceffd2386e7c72b432c9dfad6355a885 2024-11-24T04:25:50,952 DEBUG [RS_OPEN_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-11-24T04:25:50,954 DEBUG [RS_OPEN_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1093): writing seq id for ceffd2386e7c72b432c9dfad6355a885 2024-11-24T04:25:50,955 INFO [RS_OPEN_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1114): Opened ceffd2386e7c72b432c9dfad6355a885; next sequenceid=18; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63259359, jitterRate=-0.05736209452152252}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-11-24T04:25:50,955 DEBUG [RS_OPEN_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1122): Running coprocessor post-open hooks for ceffd2386e7c72b432c9dfad6355a885 2024-11-24T04:25:50,956 DEBUG [RS_OPEN_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1006): Region open journal for ceffd2386e7c72b432c9dfad6355a885: Running coprocessor pre-open hook at 1732422350928Writing region info on filesystem at 1732422350928Initializing all the Stores at 1732422350929 (+1 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732422350929Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732422350929Cleaning up temporary data from old regions at 1732422350951 (+22 ms)Running coprocessor post-open hooks at 1732422350955 (+4 ms)Region opened successfully at 1732422350956 (+1 ms) 2024-11-24T04:25:50,957 INFO [RS_OPEN_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1732422348811.ceffd2386e7c72b432c9dfad6355a885., pid=12, masterSystemTime=1732422350923 2024-11-24T04:25:50,960 DEBUG [RS_OPEN_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1732422348811.ceffd2386e7c72b432c9dfad6355a885. 2024-11-24T04:25:50,960 INFO [RS_OPEN_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1732422348811.ceffd2386e7c72b432c9dfad6355a885. 2024-11-24T04:25:50,961 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=ceffd2386e7c72b432c9dfad6355a885, regionState=OPEN, openSeqNum=18, regionLocation=fadb2fe0ba32,43811,1732422333705 2024-11-24T04:25:50,967 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=12, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure ceffd2386e7c72b432c9dfad6355a885, server=fadb2fe0ba32,43811,1732422333705 because future has completed 2024-11-24T04:25:50,968 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33969 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=fadb2fe0ba32,43811,1732422333705, table=testReplayEditsAfterRegionMovedWithMultiCF, region=ceffd2386e7c72b432c9dfad6355a885. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T04:25:50,972 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=10 2024-11-24T04:25:50,972 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=10, state=SUCCESS, hasLock=false; OpenRegionProcedure ceffd2386e7c72b432c9dfad6355a885, server=fadb2fe0ba32,43811,1732422333705 in 199 msec 2024-11-24T04:25:50,974 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=ceffd2386e7c72b432c9dfad6355a885, REOPEN/MOVE in 558 msec 2024-11-24T04:25:51,018 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-24T04:25:51,020 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33858, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-24T04:25:51,022 ERROR [Time-limited test {}] regionserver.HRegionServer(2442): ***** ABORTING region server fadb2fe0ba32,43811,1732422333705: testing ***** 2024-11-24T04:25:51,022 ERROR [Time-limited test {}] regionserver.HRegionServer(2447): RegionServer abort: loaded coprocessors are: [org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint] 2024-11-24T04:25:51,024 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for java.lang:type=Memory 2024-11-24T04:25:51,025 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=IPC 2024-11-24T04:25:51,028 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Replication 2024-11-24T04:25:51,029 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Server 2024-11-24T04:25:51,037 INFO [Time-limited test {}] regionserver.HRegionServer(2451): Dump of metrics as JSON on abort: { "beans": [ { "name": "java.lang:type=Memory", "modelerType": "sun.management.MemoryImpl", "ObjectPendingFinalizationCount": 0, "HeapMemoryUsage": { "committed": 1048576000, "init": 1048576000, "max": 2306867200, "used": 339067984 }, "NonHeapMemoryUsage": { "committed": 171966464, "init": 7667712, "max": -1, "used": 169308616 }, "Verbose": false, "ObjectName": "java.lang:type=Memory" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=IPC", "modelerType": "RegionServer,sub=IPC", "tag.Context": "regionserver", "tag.Hostname": "fadb2fe0ba32", "queueSize": 0, "numCallsInGeneralQueue": 0, "numCallsInReplicationQueue": 0, "numCallsInBulkLoadQueue": 0, "numCallsInPriorityQueue": 0, "numCallsInMetaPriorityQueue": 0, "numOpenConnections": 3, "numActiveHandler": 0, "numActiveGeneralHandler": 0, "numActivePriorityHandler": 0, "numActiveReplicationHandler": 0, "numGeneralCallsDropped": 0, "numLifoModeSwitches": 0, "numCallsInWriteQueue": 0, "numActiveBulkLoadHandler": 0, "numCallsInReadQueue": 0, "numCallsInScanQueue": 0, "numActiveWriteHandler": 0, "numActiveReadHandler": 0, "numActiveScanHandler": 0, "nettyDirectMemoryUsage": 67108864, "nettyTotalPendingOutboundBytes": 0, "nettyMaxPendingOutboundBytes": 0, "receivedBytes": 2071, "exceptions.RegionMovedException": 0, "authenticationSuccesses": 0, "authorizationFailures": 0, "exceptions.requestTooBig": 0, "UnwritableTime_num_ops": 0, "UnwritableTime_min": 0, "UnwritableTime_max": 0, "UnwritableTime_mean": 0, "UnwritableTime_25th_percentile": 0, "UnwritableTime_median": 0, "UnwritableTime_75th_percentile": 0, "UnwritableTime_90th_percentile": 0, "UnwritableTime_95th_percentile": 0, "UnwritableTime_98th_percentile": 0, "UnwritableTime_99th_percentile": 0, "UnwritableTime_99.9th_percentile": 0, "exceptions.OutOfOrderScannerNextException": 0, "exceptions.rpcThrottling": 0, "exceptions.otherExceptions": 0, "ProcessCallTime_num_ops": 8, "ProcessCallTime_min": 0, "ProcessCallTime_max": 10, "ProcessCallTime_mean": 4, "ProcessCallTime_25th_percentile": 2, "ProcessCallTime_median": 5, "ProcessCallTime_75th_percentile": 7, "ProcessCallTime_90th_percentile": 9, "ProcessCallTime_95th_percentile": 9, "ProcessCallTime_98th_percentile": 9, "ProcessCallTime_99th_percentile": 9, "ProcessCallTime_99.9th_percentile": 9, "ProcessCallTime_TimeRangeCount_0-1": 8, "exceptions.callQueueTooBig": 0, "QueueCallTime_num_ops": 8, "QueueCallTime_min": 0, "QueueCallTime_max": 1, "QueueCallTime_mean": 0, "QueueCallTime_25th_percentile": 0, "QueueCallTime_median": 0, "QueueCallTime_75th_percentile": 0, "QueueCallTime_90th_percentile": 0, "QueueCallTime_95th_percentile": 0, "QueueCallTime_98th_percentile": 0, "QueueCallTime_99th_percentile": 0, "QueueCallTime_99.9th_percentile": 0, "QueueCallTime_TimeRangeCount_0-1": 8, "authenticationFailures": 0, "exceptions.multiResponseTooLarge": 0, "exceptions.callDropped": 0, "TotalCallTime_num_ops": 8, "TotalCallTime_min": 1, "TotalCallTime_max": 10, "TotalCallTime_mean": 5, "TotalCallTime_25th_percentile": 3, "TotalCallTime_median": 5, "TotalCallTime_75th_percentile": 7, "TotalCallTime_90th_percentile": 9, "TotalCallTime_95th_percentile": 9, "TotalCallTime_98th_percentile": 9, "TotalCallTime_99th_percentile": 9, "TotalCallTime_99.9th_percentile": 9, "TotalCallTime_TimeRangeCount_0-1": 8, "exceptions.RegionTooBusyException": 0, "exceptions.FailedSanityCheckException": 0, "ResponseSize_num_ops": 8, "ResponseSize_min": 0, "ResponseSize_max": 174, "ResponseSize_mean": 50, "ResponseSize_25th_percentile": 43, "ResponseSize_median": 87, "ResponseSize_75th_percentile": 130, "ResponseSize_90th_percentile": 156, "ResponseSize_95th_percentile": 165, "ResponseSize_98th_percentile": 170, "ResponseSize_99th_percentile": 172, "ResponseSize_99.9th_percentile": 173, "ResponseSize_SizeRangeCount_0-10": 8, "exceptions.UnknownScannerException": 0, "exceptions": 0, "maxOutboundBytesExceeded": 0, "authenticationFallbacks": 0, "exceptions.quotaExceeded": 0, "exceptions.callTimedOut": 0, "exceptions.NotServingRegionException": 0, "authorizationSuccesses": 0, "exceptions.ScannerResetException": 0, "RequestSize_num_ops": 8, "RequestSize_min": 89, "RequestSize_max": 121, "RequestSize_mean": 103, "RequestSize_25th_percentile": 97, "RequestSize_median": 105, "RequestSize_75th_percentile": 113, "RequestSize_90th_percentile": 117, "RequestSize_95th_percentile": 119, "RequestSize_98th_percentile": 120, "RequestSize_99th_percentile": 120, "RequestSize_99.9th_percentile": 120, "RequestSize_SizeRangeCount_0-10": 8, "sentBytes": 348 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Replication", "modelerType": "RegionServer,sub=Replication", "tag.Context": "regionserver", "tag.Hostname": "fadb2fe0ba32", "source.shippedHFiles": 0, "Source.ageOfLastShippedOp_num_ops": 0, "Source.ageOfLastShippedOp_min": 0, "Source.ageOfLastShippedOp_max": 0, "Source.ageOfLastShippedOp_mean": 0, "Source.ageOfLastShippedOp_25th_percentile": 0, "Source.ageOfLastShippedOp_median": 0, "Source.ageOfLastShippedOp_75th_percentile": 0, "Source.ageOfLastShippedOp_90th_percentile": 0, "Source.ageOfLastShippedOp_95th_percentile": 0, "Source.ageOfLastShippedOp_98th_percentile": 0, "Source.ageOfLastShippedOp_99th_percentile": 0, "Source.ageOfLastShippedOp_99.9th_percentile": 0, "source.uncleanlyClosedLogs": 0, "source.closedLogsWithUnknownFileLength": 0, "source.walReaderEditsBufferUsage": 0, "source.repeatedLogFileBytes": 0, "source.sizeOfHFileRefsQueue": 0, "source.logReadInBytes": 0, "source.completedRecoverQueues": 0, "source.sizeOfLogQueue": 0, "source.restartedLogReading": 0, "source.failedRecoverQueues": 0, "source.ignoredUncleanlyClosedLogContentsInBytes": 0, "Sink.ageOfLastAppliedOp_num_ops": 0, "Sink.ageOfLastAppliedOp_min": 0, "Sink.ageOfLastAppliedOp_max": 0, "Sink.ageOfLastAppliedOp_mean": 0, "Sink.ageOfLastAppliedOp_25th_percentile": 0, "Sink.ageOfLastAppliedOp_median": 0, "Sink.ageOfLastAppliedOp_75th_percentile": 0, "Sink.ageOfLastAppliedOp_90th_percentile": 0, "Sink.ageOfLastAppliedOp_95th_percentile": 0, "Sink.ageOfLastAppliedOp_98th_percentile": 0, "Sink.ageOfLastAppliedOp_99th_percentile": 0, "Sink.ageOfLastAppliedOp_99.9th_percentile": 0, "source.logEditsRead": 0, "source.numInitializing": 0, "source.shippedOps": 0, "sink.appliedHFiles": 0, "source.logEditsFiltered": 0, "source.shippedBytes": 0, "sink.appliedOps": 0, "source.completedLogs": 0, "source.failedBatches": 0, "sink.failedBatches": 0, "source.shippedBatches": 0, "sink.appliedBatches": 0 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Server", "modelerType": "RegionServer,sub=Server", "tag.zookeeperQuorum": "127.0.0.1:62017", "tag.serverName": "fadb2fe0ba32,43811,1732422333705", "tag.clusterId": "19ccd069-e1d9-4899-ace9-8d275717a5ba", "tag.Context": "regionserver", "tag.Hostname": "fadb2fe0ba32", "regionCount": 0, "storeCount": 0, "hlogFileCount": 1, "hlogFileSize": 0, "storeFileCount": 0, "maxStoreFileCount": 0, "memStoreSize": 0, "memStoreHeapSize": 0, "memStoreOffHeapSize": 0, "storeFileSize": 0, "storeFileSizeGrowthRate": 0.0, "maxStoreFileAge": 0, "minStoreFileAge": 0, "avgStoreFileAge": 0, "numReferenceFiles": 0, "regionServerStartTime": 1732422333705, "averageRegionSize": 0, "storeFileIndexSize": 0, "staticIndexSize": 0, "staticBloomSize": 0, "bloomFilterRequestsCount": 0, "bloomFilterNegativeResultsCount": 0, "bloomFilterEligibleRequestsCount": 0, "mutationsWithoutWALCount": 0, "mutationsWithoutWALSize": 0, "percentFilesLocal": 0.0, "percentFilesLocalSecondaryRegions": 0.0, "totalBytesRead": 10034, "localBytesRead": 10034, "shortCircuitBytesRead": 0, "zeroCopyBytesRead": 0, "splitQueueLength": 0, "compactionQueueLength": 0, "smallCompactionQueueLength": 0, "largeCompactionQueueLength": 0, "flushQueueLength": 0, "blockCacheFreeSize": 922070024, "blockCacheCount": 0, "blockCacheDataBlockCount": 0, "blockCacheSize": 676856, "blockCacheCountHitPercent": 0.0, "blockCacheExpressHitPercent": 0.0, "l1CacheSize": 676856, "l1CacheFreeSize": 922070024, "l1CacheCount": 0, "l1CacheEvictionCount": 0, "l1CacheHitCount": 0, "l1CacheMissCount": 0, "l1CacheHitRatio": 0.0, "l1CacheMissRatio": 0.0, "l2CacheSize": 0, "l2CacheFreeSize": 0, "l2CacheCount": 0, "l2CacheEvictionCount": 0, "l2CacheHitCount": 0, "l2CacheMissCount": 0, "l2CacheHitRatio": 0.0, "l2CacheMissRatio": 0.0, "mobFileCacheCount": 0, "mobFileCacheHitPercent": 0.0, "readRequestRatePerSecond": 0.0, "writeRequestRatePerSecond": 0.0, "ByteBuffAllocatorHeapAllocationBytes": 4782, "ByteBuffAllocatorPoolAllocationBytes": 0, "ByteBuffAllocatorHeapAllocationRatio": 0.0, "ByteBuffAllocatorTotalBufferCount": 186, "ByteBuffAllocatorUsedBufferCount": 0, "activeScanners": 0, "totalRequestCount": 0, "totalRowActionRequestCount": 0, "readRequestCount": 0, "cpRequestCount": 0, "filteredReadRequestCount": 0, "writeRequestCount": 0, "rpcGetRequestCount": 0, "rpcFullScanRequestCount": 0, "rpcScanRequestCount": 0, "rpcMultiRequestCount": 0, "rpcMutateRequestCount": 0, "checkMutateFailedCount": 0, "checkMutatePassedCount": 0, "blockCacheHitCount": 0, "blockCacheHitCountPrimary": 0, "blockCacheHitCachingCount": 0, "blockCacheMissCount": 0, "blockCacheMissCountPrimary": 0, "blockCacheMissCachingCount": 0, "blockCacheEvictionCount": 0, "blockCacheEvictionCountPrimary": 0, "blockCacheFailedInsertionCount": 0, "blockCacheDataMissCount": 0, "blockCacheLeafIndexMissCount": 0, "blockCacheBloomChunkMissCount": 0, "blockCacheMetaMissCount": 0, "blockCacheRootIndexMissCount": 0, "blockCacheIntermediateIndexMissCount": 0, "blockCacheFileInfoMissCount": 0, "blockCacheGeneralBloomMetaMissCount": 0, "blockCacheDeleteFamilyBloomMissCount": 0, "blockCacheTrailerMissCount": 0, "blockCacheDataHitCount": 0, "blockCacheLeafIndexHitCount": 0, "blockCacheBloomChunkHitCount": 0, "blockCacheMetaHitCount": 0, "blockCacheRootIndexHitCount": 0, "blockCacheIntermediateIndexHitCount": 0, "blockCacheFileInfoHitCount": 0, "blockCacheGeneralBloomMetaHitCount": 0, "blockCacheDeleteFamilyBloomHitCount": 0, "blockCacheTrailerHitCount": 0, "updatesBlockedTime": 0, "flushedCellsCount": 0, "compactedCellsCount": 0, "majorCompactedCellsCount": 0, "flushedCellsSize": 0, "compactedCellsSize": 0, "majorCompactedCellsSize": 0, "cellsCountCompactedFromMob": 0, "cellsCountCompactedToMob": 0, "cellsSizeCompactedFromMob": 0, "cellsSizeCompactedToMob": 0, "mobFlushCount": 0, "mobFlushedCellsCount": 0, "mobFlushedCellsSize": 0, "mobScanCellsCount": 0, "mobScanCellsSize": 0, "mobFileCacheAccessCount": 0, "mobFileCacheMissCount": 0, "mobFileCacheEvictedCount": 0, "hedgedReads": 0, "hedgedReadWins": 0, "hedgedReadOpsInCurThread": 0, "blockedRequestCount": 0, "CheckAndMutate_num_ops": 0, "CheckAndMutate_min": 0, "CheckAndMutate_max": 0, "CheckAndMutate_mean": 0, "CheckAndMutate_25th_percentile": 0, "CheckAndMutate_median": 0, "CheckAndMutate_75th_percentile": 0, "CheckAndMutate_90th_percentile": 0, "CheckAndMutate_95th_percentile": 0, "CheckAndMutate_98th_percentile": 0, "CheckAndMutate_99th_percentile": 0, "CheckAndMutate_99.9th_percentile": 0, "MajorCompactionTime_num_ops": 0, "MajorCompactionTime_min": 0, "MajorCompactionTime_max": 0, "MajorCompactionTime_mean": 0, "MajorCompactionTime_25th_percentile": 0, "MajorCompactionTime_median": 0, "MajorCompactionTime_75th_percentile": 0, "MajorCompactionTime_90th_percentile": 0, "MajorCompactionTime_95th_percentile": 0, "MajorCompactionTime_98th_percentile": 0, "MajorCompactionTime_99th_percentile": 0, "MajorCompactionTime_99.9th_percentile": 0, "ScanTime_num_ops": 0, "ScanTime_min": 0, "ScanTime_max": 0, "ScanTime_mean": 0, "ScanTime_25th_percentile": 0, "ScanTime_median": 0, "ScanTime_75th_percentile": 0, "ScanTime_90th_percentile": 0, "ScanTime_95th_percentile": 0, "ScanTime_98th_percentile": 0, "ScanTime_99th_percentile": 0, "ScanTime_99.9th_percentile": 0, "CheckAndMutateBlockBytesScanned_num_ops": 0, "CheckAndMutateBlockBytesScanned_min": 0, "CheckAndMutateBlockBytesScanned_max": 0, "CheckAndMutateBlockBytesScanned_mean": 0, "CheckAndMutateBlockBytesScanned_25th_percentile": 0, "CheckAndMutateBlockBytesScanned_median": 0, "CheckAndMutateBlockBytesScanned_75th_percentile": 0, "CheckAndMutateBlockBytesScanned_90th_percentile": 0, "CheckAndMutateBlockBytesScanned_95th_percentile": 0, "CheckAndMutateBlockBytesScanned_98th_percentile": 0, "CheckAndMutateBlockBytesScanned_99th_percentile": 0, "CheckAndMutateBlockBytesScanned_99.9th_percentile": 0, "Put_num_ops": 0, "Put_min": 0, "Put_max": 0, "Put_mean": 0, "Put_25th_percentile": 0, "Put_median": 0, "Put_75th_percentile": 0, "Put_90th_percentile": 0, "Put_95th_percentile": 0, "Put_98th_percentile": 0, "Put_99th_percentile": 0, "Put_99.9th_percentile": 0, "splitRequestCount": 0, "AppendBlockBytesScanned_num_ops": 0, "AppendBlockBytesScanned_min": 0, "AppendBlockBytesScanned_max": 0, "AppendBlockBytesScanned_mean": 0, "AppendBlockBytesScanned_25th_percentile": 0, "AppendBlockBytesScanned_median": 0, "AppendBlockBytesScanned_75th_percentile": 0, "AppendBlockBytesScanned_90th_percentile": 0, "AppendBlockBytesScanned_95th_percentile": 0, "AppendBlockBytesScanned_98th_percentile": 0, "AppendBlockBytesScanned_99th_percentile": 0, "AppendBlockBytesScanned_99.9th_percentile": 0, "PutBatch_num_ops": 0, "PutBatch_min": 0, "PutBatch_max": 0, "PutBatch_mean": 0, "PutBatch_25th_percentile": 0, "PutBatch_median": 0, "PutBatch_75th_percentile": 0, "PutBatch_90th_percentile": 0, "PutBatch_95th_percentile": 0, "PutBatch_98th_percentile": 0, "PutBatch_99th_percentile": 0, "PutBatch_99.9th_percentile": 0, "IncrementBlockBytesScanned_num_ops": 0, "IncrementBlockBytesScanned_min": 0, "IncrementBlockBytesScanned_max": 0, "IncrementBlockBytesScanned_mean": 0, "IncrementBlockBytesScanned_25th_percentile": 0, "IncrementBlockBytesScanned_median": 0, "IncrementBlockBytesScanned_75th_percentile": 0, "IncrementBlockBytesScanned_90th_percentile": 0, "IncrementBlockBytesScanned_95th_percentile": 0, "IncrementBlockBytesScanned_98th_percentile": 0, "IncrementBlockBytesScanned_99th_percentile": 0, "IncrementBlockBytesScanned_99.9th_percentile": 0, "SplitTime_num_ops": 0, "SplitTime_min": 0, "SplitTime_max": 0, "SplitTime_mean": 0, "SplitTime_25th_percentile": 0, "SplitTime_median": 0, "SplitTime_75th_percentile": 0, "SplitTime_90th_percentile": 0, "SplitTime_95th_percentile": 0, "SplitTime_98th_percentile": 0, "SplitTime_99th_percentile": 0, "SplitTime_99.9th_percentile": 0, "GetBlockBytesScanned_num_ops": 0, "GetBlockBytesScanned_min": 0, "GetBlockBytesScanned_max": 0, "GetBlockBytesScanned_mean": 0, "GetBlockBytesScanned_25th_percentile": 0, "GetBlockBytesScanned_median": 0, "GetBlockBytesScanned_75th_percentile": 0, "GetBlockBytesScanned_90th_percentile": 0, "GetBlockBytesScanned_95th_percentile": 0, "GetBlockBytesScanned_98th_percentile": 0, "GetBlockBytesScanned_99th_percentile": 0, "GetBlockBytesScanned_99.9th_percentile": 0, "majorCompactedInputBytes": 0, "slowAppendCount": 0, "flushedOutputBytes": 0, "Replay_num_ops": 0, "Replay_min": 0, "Replay_max": 0, "Replay_mean": 0, "Replay_25th_percentile": 0, "Replay_median": 0, "Replay_75th_percentile": 0, "Replay_90th_percentile": 0, "Replay_95th_percentile": 0, "Replay_98th_percentile": 0, "Replay_99th_percentile": 0, "Replay_99.9th_percentile": 0, "MajorCompactionInputSize_num_ops": 0, "MajorCompactionInputSize_min": 0, "MajorCompactionInputSize_max": 0, "MajorCompactionInputSize_mean": 0, "MajorCompactionInputSize_25th_percentile": 0, "MajorCompactionInputSize_median": 0, "MajorCompactionInputSize_75th_percentile": 0, "MajorCompactionInputSize_90th_percentile": 0, "MajorCompactionInputSize_95th_percentile": 0, "MajorCompactionInputSize_98th_percentile": 0, "MajorCompactionInputSize_99th_percentile": 0, "MajorCompactionInputSize_99.9th_percentile": 0, "pauseInfoThresholdExceeded": 0, "CheckAndDelete_num_ops": 0, "CheckAndDelete_min": 0, "CheckAndDelete_max": 0, "CheckAndDelete_mean": 0, "CheckAndDelete_25th_percentile": 0, "CheckAndDelete_median": 0, "CheckAndDelete_75th_percentile": 0, "CheckAndDelete_90th_percentile": 0, "CheckAndDelete_95th_percentile": 0, "CheckAndDelete_98th_percentile": 0, "CheckAndDelete_99th_percentile": 0, "CheckAndDelete_99.9th_percentile": 0, "CompactionInputSize_num_ops": 0, "CompactionInputSize_min": 0, "CompactionInputSize_max": 0, "CompactionInputSize_mean": 0, "CompactionInputSize_25th_percentile": 0, "CompactionInputSize_median": 0, "CompactionInputSize_75th_percentile": 0, "CompactionInputSize_90th_percentile": 0, "CompactionInputSize_95th_percentile": 0, "CompactionInputSize_98th_percentile": 0, "CompactionInputSize_99th_percentile": 0, "CompactionInputSize_99.9th_percentile": 0, "flushedMemstoreBytes": 0, "majorCompactedOutputBytes": 0, "slowPutCount": 0, "compactedInputBytes": 0, "FlushOutputSize_num_ops": 0, "FlushOutputSize_min": 0, "FlushOutputSize_max": 0, "FlushOutputSize_mean": 0, "FlushOutputSize_25th_percentile": 0, "FlushOutputSize_median": 0, "FlushOutputSize_75th_percentile": 0, "FlushOutputSize_90th_percentile": 0, "FlushOutputSize_95th_percentile": 0, "FlushOutputSize_98th_percentile": 0, "FlushOutputSize_99th_percentile": 0, "FlushOutputSize_99.9th_percentile": 0, "PauseTimeWithGc_num_ops": 0, "PauseTimeWithGc_min": 0, "PauseTimeWithGc_max": 0, "PauseTimeWithGc_mean": 0, "PauseTimeWithGc_25th_percentile": 0, "PauseTimeWithGc_median": 0, "PauseTimeWithGc_75th_percentile": 0, "PauseTimeWithGc_90th_percentile": 0, "PauseTimeWithGc_95th_percentile": 0, "PauseTimeWithGc_98th_percentile": 0, "PauseTimeWithGc_99th_percentile": 0, "PauseTimeWithGc_99.9th_percentile": 0, "compactedOutputBytes": 0, "pauseWarnThresholdExceeded": 0, "ScanBlockBytesScanned_num_ops": 0, "ScanBlockBytesScanned_min": 0, "ScanBlockBytesScanned_max": 0, "ScanBlockBytesScanned_mean": 0, "ScanBlockBytesScanned_25th_percentile": 0, "ScanBlockBytesScanned_median": 0, "ScanBlockBytesScanned_75th_percentile": 0, "ScanBlockBytesScanned_90th_percentile": 0, "ScanBlockBytesScanned_95th_percentile": 0, "ScanBlockBytesScanned_98th_percentile": 0, "ScanBlockBytesScanned_99th_percentile": 0, "ScanBlockBytesScanned_99.9th_percentile": 0, "Increment_num_ops": 0, "Increment_min": 0, "Increment_max": 0, "Increment_mean": 0, "Increment_25th_percentile": 0, "Increment_median": 0, "Increment_75th_percentile": 0, "Increment_90th_percentile": 0, "Increment_95th_percentile": 0, "Increment_98th_percentile": 0, "Increment_99th_percentile": 0, "Increment_99.9th_percentile": 0, "Delete_num_ops": 0, "Delete_min": 0, "Delete_max": 0, "Delete_mean": 0, "Delete_25th_percentile": 0, "Delete_median": 0, "Delete_75th_percentile": 0, "Delete_90th_percentile": 0, "Delete_95th_percentile": 0, "Delete_98th_percentile": 0, "Delete_99th_percentile": 0, "Delete_99.9th_percentile": 0, "DeleteBatch_num_ops": 0, "DeleteBatch_min": 0, "DeleteBatch_max": 0, "DeleteBatch_mean": 0, "DeleteBatch_25th_percentile": 0, "DeleteBatch_median": 0, "DeleteBatch_75th_percentile": 0, "DeleteBatch_90th_percentile": 0, "DeleteBatch_95th_percentile": 0, "DeleteBatch_98th_percentile": 0, "DeleteBatch_99th_percentile": 0, "DeleteBatch_99.9th_percentile": 0, "blockBytesScannedCount": 0, "FlushMemstoreSize_num_ops": 0, "FlushMemstoreSize_min": 0, "FlushMemstoreSize_max": 0, "FlushMemstoreSize_mean": 0, "FlushMemstoreSize_25th_percentile": 0, "FlushMemstoreSize_median": 0, "FlushMemstoreSize_75th_percentile": 0, "FlushMemstoreSize_90th_percentile": 0, "FlushMemstoreSize_95th_percentile": 0, "FlushMemstoreSize_98th_percentile": 0, "FlushMemstoreSize_99th_percentile": 0, "FlushMemstoreSize_99.9th_percentile": 0, "CompactionInputFileCount_num_ops": 0, "CompactionInputFileCount_min": 0, "CompactionInputFileCount_max": 0, "CompactionInputFileCount_mean": 0, "CompactionInputFileCount_25th_percentile": 0, "CompactionInputFileCount_median": 0, "CompactionInputFileCount_75th_percentile": 0, "CompactionInputFileCount_90th_percentile": 0, "CompactionInputFileCount_95th_percentile": 0, "CompactionInputFileCount_98th_percentile": 0, "CompactionInputFileCount_99th_percentile": 0, "CompactionInputFileCount_99.9th_percentile": 0, "CompactionTime_num_ops": 0, "CompactionTime_min": 0, "CompactionTime_max": 0, "CompactionTime_mean": 0, "CompactionTime_25th_percentile": 0, "CompactionTime_median": 0, "CompactionTime_75th_percentile": 0, "CompactionTime_90th_percentile": 0, "CompactionTime_95th_percentile": 0, "CompactionTime_98th_percentile": 0, "CompactionTime_99th_percentile": 0, "CompactionTime_99.9th_percentile": 0, "Get_num_ops": 0, "Get_min": 0, "Get_max": 0, "Get_mean": 0, "Get_25th_percentile": 0, "Get_median": 0, "Get_75th_percentile": 0, "Get_90th_percentile": 0, "Get_95th_percentile": 0, "Get_98th_percentile": 0, "Get_99th_percentile": 0, "Get_99.9th_percentile": 0, "MajorCompactionInputFileCount_num_ops": 0, "MajorCompactionInputFileCount_min": 0, "MajorCompactionInputFileCount_max": 0, "MajorCompactionInputFileCount_mean": 0, "MajorCompactionInputFileCount_25th_percentile": 0, "MajorCompactionInputFileCount_median": 0, "MajorCompactionInputFileCount_75th_percentile": 0, "MajorCompactionInputFileCount_90th_percentile": 0, "MajorCompactionInputFileCount_95th_percentile": 0, "MajorCompactionInputFileCount_98th_percentile": 0, "MajorCompactionInputFileCount_99th_percentile": 0, "MajorCompactionInputFileCount_99.9th_percentile": 0, "scannerLeaseExpiredCount": 0, "CheckAndPut_num_ops": 0, "CheckAndPut_min": 0, "CheckAndPut_max": 0, "CheckAndPut_mean": 0, "CheckAndPut_25th_percentile": 0, "CheckAndPut_median": 0, "CheckAndPut_75th_percentile": 0, "CheckAndPut_90th_percentile": 0, "CheckAndPut_95th_percentile": 0, "CheckAndPut_98th_percentile": 0, "CheckAndPut_99th_percentile": 0, "CheckAndPut_99.9th_percentile": 0, "MajorCompactionOutputSize_num_ops": 0, "MajorCompactionOutputSize_min": 0, "MajorCompactionOutputSize_max": 0, "MajorCompactionOutputSize_mean": 0, "MajorCompactionOutputSize_25th_percentile": 0, "MajorCompactionOutputSize_median": 0, "MajorCompactionOutputSize_75th_percentile": 0, "MajorCompactionOutputSize_90th_percentile": 0, "MajorCompactionOutputSize_95th_percentile": 0, "MajorCompactionOutputSize_98th_percentile": 0, "MajorCompactionOutputSize_99th_percentile": 0, "MajorCompactionOutputSize_99.9th_percentile": 0, "CompactionOutputFileCount_num_ops": 0, "CompactionOutputFileCount_min": 0, "CompactionOutputFileCount_max": 0, "CompactionOutputFileCount_mean": 0, "CompactionOutputFileCount_25th_percentile": 0, "CompactionOutputFileCount_median": 0, "CompactionOutputFileCount_75th_percentile": 0, "CompactionOutputFileCount_90th_percentile": 0, "CompactionOutputFileCount_95th_percentile": 0, "CompactionOutputFileCount_98th_percentile": 0, "CompactionOutputFileCount_99th_percentile": 0, "CompactionOutputFileCount_99.9th_percentile": 0, "slowDeleteCount": 0, "FlushTime_num_ops": 0, "FlushTime_min": 0, "FlushTime_max": 0, "FlushTime_mean": 0, "FlushTime_25th_percentile": 0, "FlushTime_median": 0, "FlushTime_75th_percentile": 0, "FlushTime_90th_percentile": 0, "FlushTime_95th_percentile": 0, "FlushTime_98th_percentile": 0, "FlushTime_99th_percentile": 0, "FlushTime_99.9th_percentile": 0, "splitSuccessCount": 0, "MajorCompactionOutputFileCount_num_ops": 0, "MajorCompactionOutputFileCount_min": 0, "MajorCompactionOutputFileCount_max": 0, "MajorCompactionOutputFileCount_mean": 0, "MajorCompactionOutputFileCount_25th_percentile": 0, "MajorCompactionOutputFileCount_median": 0, "MajorCompactionOutputFileCount_75th_percentile": 0, "MajorCompactionOutputFileCount_90th_percentile": 0, "MajorCompactionOutputFileCount_95th_percentile": 0, "MajorCompactionOutputFileCount_98th_percentile": 0, "MajorCompactionOutputFileCount_99th_percentile": 0, "MajorCompactionOutputFileCount_99.9th_percentile": 0, "slowGetCount": 0, "ScanSize_num_ops": 0, "ScanSize_min": 0, "ScanSize_max": 0, "ScanSize_mean": 0, "ScanSize_25th_percentile": 0, "ScanSize_median": 0, "ScanSize_75th_percentile": 0, "ScanSize_90th_percentile": 0, "ScanSize_95th_percentile": 0, "ScanSize_98th_percentile": 0, "ScanSize_99th_percentile": 0, "ScanSize_99.9th_percentile": 0, "CompactionOutputSize_num_ops": 0, "CompactionOutputSize_min": 0, "CompactionOutputSize_max": 0, "CompactionOutputSize_mean": 0, "CompactionOutputSize_25th_percentile": 0, "CompactionOutputSize_median": 0, "CompactionOutputSize_75th_percentile": 0, "CompactionOutputSize_90th_percentile": 0, "CompactionOutputSize_95th_percentile": 0, "CompactionOutputSize_98th_percentile": 0, "CompactionOutputSize_99th_percentile": 0, "CompactionOutputSize_99.9th_percentile": 0, "PauseTimeWithoutGc_num_ops": 0, "PauseTimeWithoutGc_min": 0, "PauseTimeWithoutGc_max": 0, "PauseTimeWithoutGc_mean": 0, "PauseTimeWithoutGc_25th_percentile": 0, "PauseTimeWithoutGc_median": 0, "PauseTimeWithoutGc_75th_percentile": 0, "PauseTimeWithoutGc_90th_percentile": 0, "PauseTimeWithoutGc_95th_percentile": 0, "PauseTimeWithoutGc_98th_percentile": 0, "PauseTimeWithoutGc_99th_percentile": 0, "PauseTimeWithoutGc_99.9th_percentile": 0, "slowIncrementCount": 0, "Append_num_ops": 0, "Append_min": 0, "Append_max": 0, "Append_mean": 0, "Append_25th_percentile": 0, "Append_median": 0, "Append_75th_percentile": 0, "Append_90th_percentile": 0, "Append_95th_percentile": 0, "Append_98th_percentile": 0, "Append_99th_percentile": 0, "Append_99.9th_percentile": 0, "Bulkload_count": 0, "Bulkload_mean_rate": 0.0, "Bulkload_1min_rate": 0.0, "Bulkload_5min_rate": 0.0, "Bulkload_15min_rate": 0.0, "Bulkload_num_ops": 0, "Bulkload_min": 0, "Bulkload_max": 0, "Bulkload_mean": 0, "Bulkload_25th_percentile": 0, "Bulkload_median": 0, "Bulkload_75th_percentile": 0, "Bulkload_90th_percentile": 0, "Bulkload_95th_percentile": 0, "Bulkload_98th_percentile": 0, "Bulkload_99th_percentile": 0, "Bulkload_99.9th_percentile": 0 } ] } 2024-11-24T04:25:51,041 WARN [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33969 {}] master.MasterRpcServices(700): fadb2fe0ba32,43811,1732422333705 reported a fatal error: ***** ABORTING region server fadb2fe0ba32,43811,1732422333705: testing ***** 2024-11-24T04:25:51,043 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'fadb2fe0ba32,43811,1732422333705' ***** 2024-11-24T04:25:51,043 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: testing 2024-11-24T04:25:51,044 INFO [RS:1;fadb2fe0ba32:43811 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-24T04:25:51,044 INFO [RS:1;fadb2fe0ba32:43811 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager abruptly. 2024-11-24T04:25:51,044 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-24T04:25:51,044 INFO [RS:1;fadb2fe0ba32:43811 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager abruptly. 2024-11-24T04:25:51,044 INFO [RS:1;fadb2fe0ba32:43811 {}] regionserver.HRegionServer(3091): Received CLOSE for ceffd2386e7c72b432c9dfad6355a885 2024-11-24T04:25:51,045 INFO [RS:1;fadb2fe0ba32:43811 {}] regionserver.HRegionServer(956): aborting server fadb2fe0ba32,43811,1732422333705 2024-11-24T04:25:51,045 INFO [RS:1;fadb2fe0ba32:43811 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-24T04:25:51,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35977 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Get size: 140 connection: 172.17.0.2:48946 deadline: 1732422411044, exception=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=fadb2fe0ba32 port=43811 startCode=1732422333705. As of locationSeqNum=12. 2024-11-24T04:25:51,045 DEBUG [RS_CLOSE_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing ceffd2386e7c72b432c9dfad6355a885, disabling compactions & flushes 2024-11-24T04:25:51,045 INFO [RS_CLOSE_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1732422348811.ceffd2386e7c72b432c9dfad6355a885. 2024-11-24T04:25:51,045 INFO [RS:1;fadb2fe0ba32:43811 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;fadb2fe0ba32:43811. 2024-11-24T04:25:51,045 DEBUG [RS_CLOSE_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732422348811.ceffd2386e7c72b432c9dfad6355a885. 2024-11-24T04:25:51,045 DEBUG [RS_CLOSE_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732422348811.ceffd2386e7c72b432c9dfad6355a885. after waiting 0 ms 2024-11-24T04:25:51,045 DEBUG [RS_CLOSE_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1732422348811.ceffd2386e7c72b432c9dfad6355a885. 2024-11-24T04:25:51,045 DEBUG [RS:1;fadb2fe0ba32:43811 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-24T04:25:51,046 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732422348811.ceffd2386e7c72b432c9dfad6355a885., hostname=fadb2fe0ba32,35977,1732422333601, seqNum=5 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1732422348811.ceffd2386e7c72b432c9dfad6355a885., hostname=fadb2fe0ba32,35977,1732422333601, seqNum=5, error=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=fadb2fe0ba32 port=43811 startCode=1732422333705. As of locationSeqNum=12. 2024-11-24T04:25:51,046 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732422348811.ceffd2386e7c72b432c9dfad6355a885., hostname=fadb2fe0ba32,35977,1732422333601, seqNum=5 is org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=fadb2fe0ba32 port=43811 startCode=1732422333705. As of locationSeqNum=12. 2024-11-24T04:25:51,046 DEBUG [RS:1;fadb2fe0ba32:43811 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T04:25:51,046 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(84): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732422348811.ceffd2386e7c72b432c9dfad6355a885., hostname=fadb2fe0ba32,35977,1732422333601, seqNum=5 with the new location region=testReplayEditsAfterRegionMovedWithMultiCF,,1732422348811.ceffd2386e7c72b432c9dfad6355a885., hostname=fadb2fe0ba32,43811,1732422333705, seqNum=12 constructed by org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=fadb2fe0ba32 port=43811 startCode=1732422333705. As of locationSeqNum=12. 2024-11-24T04:25:51,046 INFO [RS:1;fadb2fe0ba32:43811 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-24T04:25:51,046 DEBUG [RS:1;fadb2fe0ba32:43811 {}] regionserver.HRegionServer(1325): Online Regions={ceffd2386e7c72b432c9dfad6355a885=testReplayEditsAfterRegionMovedWithMultiCF,,1732422348811.ceffd2386e7c72b432c9dfad6355a885.} 2024-11-24T04:25:51,047 DEBUG [RS:1;fadb2fe0ba32:43811 {}] regionserver.HRegionServer(1351): Waiting on ceffd2386e7c72b432c9dfad6355a885 2024-11-24T04:25:51,048 INFO [RS_CLOSE_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732422348811.ceffd2386e7c72b432c9dfad6355a885. 2024-11-24T04:25:51,048 DEBUG [RS_CLOSE_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for ceffd2386e7c72b432c9dfad6355a885: Waiting for close lock at 1732422351045Running coprocessor pre-close hooks at 1732422351045Disabling compacts and flushes for region at 1732422351045Disabling writes for close at 1732422351045Writing region close event to WAL at 1732422351047 (+2 ms)Running coprocessor post-close hooks at 1732422351047Closed at 1732422351048 (+1 ms) 2024-11-24T04:25:51,048 DEBUG [RS_CLOSE_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732422348811.ceffd2386e7c72b432c9dfad6355a885. 2024-11-24T04:25:51,053 INFO [regionserver/fadb2fe0ba32:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-24T04:25:51,053 INFO [regionserver/fadb2fe0ba32:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-24T04:25:51,073 INFO [regionserver/fadb2fe0ba32:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-24T04:25:51,158 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server fadb2fe0ba32,43811,1732422333705 aborting at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processRequest(ServerRpcConnection.java:564) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processOneRpc(ServerRpcConnection.java:364) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyServerRpcConnection.process(NettyServerRpcConnection.java:89) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:56) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:31) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T04:25:51,159 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732422348811.ceffd2386e7c72b432c9dfad6355a885., hostname=fadb2fe0ba32,43811,1732422333705, seqNum=12 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1732422348811.ceffd2386e7c72b432c9dfad6355a885., hostname=fadb2fe0ba32,43811,1732422333705, seqNum=12, error=org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server fadb2fe0ba32,43811,1732422333705 aborting 2024-11-24T04:25:51,160 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732422348811.ceffd2386e7c72b432c9dfad6355a885., hostname=fadb2fe0ba32,43811,1732422333705, seqNum=12 is org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server fadb2fe0ba32,43811,1732422333705 aborting 2024-11-24T04:25:51,160 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testReplayEditsAfterRegionMovedWithMultiCF,,1732422348811.ceffd2386e7c72b432c9dfad6355a885., hostname=fadb2fe0ba32,43811,1732422333705, seqNum=12 from cache 2024-11-24T04:25:51,247 INFO [RS:1;fadb2fe0ba32:43811 {}] regionserver.HRegionServer(976): stopping server fadb2fe0ba32,43811,1732422333705; all regions closed. 2024-11-24T04:25:51,250 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44781 is added to blk_1073741835_1011 (size=1407) 2024-11-24T04:25:51,251 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34457 is added to blk_1073741835_1011 (size=1407) 2024-11-24T04:25:51,251 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36069 is added to blk_1073741835_1011 (size=1407) 2024-11-24T04:25:51,251 DEBUG [RS:1;fadb2fe0ba32:43811 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T04:25:51,251 INFO [RS:1;fadb2fe0ba32:43811 {}] regionserver.LeaseManager(133): Closed leases 2024-11-24T04:25:51,252 INFO [RS:1;fadb2fe0ba32:43811 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-24T04:25:51,252 INFO [RS:1;fadb2fe0ba32:43811 {}] hbase.ChoreService(370): Chore service for: regionserver/fadb2fe0ba32:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-24T04:25:51,252 INFO [regionserver/fadb2fe0ba32:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-24T04:25:51,252 INFO [RS:1;fadb2fe0ba32:43811 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-24T04:25:51,252 INFO [RS:1;fadb2fe0ba32:43811 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-24T04:25:51,252 INFO [RS:1;fadb2fe0ba32:43811 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-24T04:25:51,252 INFO [RS:1;fadb2fe0ba32:43811 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-24T04:25:51,253 INFO [RS:1;fadb2fe0ba32:43811 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:43811 2024-11-24T04:25:51,260 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43811-0x1014838d9450002, quorum=127.0.0.1:62017, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/fadb2fe0ba32,43811,1732422333705 2024-11-24T04:25:51,260 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33969-0x1014838d9450000, quorum=127.0.0.1:62017, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-24T04:25:51,260 INFO [RS:1;fadb2fe0ba32:43811 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-24T04:25:51,262 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [fadb2fe0ba32,43811,1732422333705] 2024-11-24T04:25:51,263 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/fadb2fe0ba32,43811,1732422333705 already deleted, retry=false 2024-11-24T04:25:51,263 INFO [RegionServerTracker-0 {}] master.ServerManager(695): Processing expiration of fadb2fe0ba32,43811,1732422333705 on fadb2fe0ba32,33969,1732422332893 2024-11-24T04:25:51,268 DEBUG [RegionServerTracker-0 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure fadb2fe0ba32,43811,1732422333705, splitWal=true, meta=false 2024-11-24T04:25:51,271 INFO [RegionServerTracker-0 {}] assignment.AssignmentManager(1999): Scheduled ServerCrashProcedure pid=13 for fadb2fe0ba32,43811,1732422333705 (carryingMeta=false) fadb2fe0ba32,43811,1732422333705/CRASHED/regionCount=1/lock=java.util.concurrent.locks.ReentrantReadWriteLock@1ae4041c[Write locks = 1, Read locks = 0], oldState=ONLINE. 2024-11-24T04:25:51,271 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(169): Start pid=13, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure fadb2fe0ba32,43811,1732422333705, splitWal=true, meta=false 2024-11-24T04:25:51,273 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(207): fadb2fe0ba32,43811,1732422333705 had 1 regions 2024-11-24T04:25:51,274 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=13, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure fadb2fe0ba32,43811,1732422333705, splitWal=true, meta=false, isMeta: false 2024-11-24T04:25:51,276 DEBUG [PEWorker-1 {}] master.MasterWalManager(329): Renamed region directory: hdfs://localhost:44611/user/jenkins/test-data/33c38499-d2fe-b6f0-8d8b-5809759afd19/WALs/fadb2fe0ba32,43811,1732422333705-splitting 2024-11-24T04:25:51,277 INFO [PEWorker-1 {}] master.SplitWALManager(105): fadb2fe0ba32,43811,1732422333705 WAL count=1, meta=false 2024-11-24T04:25:51,280 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure fadb2fe0ba32%2C43811%2C1732422333705.1732422335221}] 2024-11-24T04:25:51,285 DEBUG [PEWorker-2 {}] master.SplitWALManager(158): Acquired split WAL worker=fadb2fe0ba32,45209,1732422333751 2024-11-24T04:25:51,287 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=15, ppid=14, state=RUNNABLE, hasLock=false; SplitWALRemoteProcedure fadb2fe0ba32%2C43811%2C1732422333705.1732422335221, worker=fadb2fe0ba32,45209,1732422333751}] 2024-11-24T04:25:51,362 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43811-0x1014838d9450002, quorum=127.0.0.1:62017, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-24T04:25:51,362 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43811-0x1014838d9450002, quorum=127.0.0.1:62017, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-24T04:25:51,363 INFO [RS:1;fadb2fe0ba32:43811 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-24T04:25:51,363 INFO [RS:1;fadb2fe0ba32:43811 {}] regionserver.HRegionServer(1031): Exiting; stopping=fadb2fe0ba32,43811,1732422333705; zookeeper connection closed. 2024-11-24T04:25:51,364 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@3fd2e883 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@3fd2e883 2024-11-24T04:25:51,372 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1732422348811.ceffd2386e7c72b432c9dfad6355a885., hostname=fadb2fe0ba32,43811,1732422333705, seqNum=18] 2024-11-24T04:25:51,374 WARN [RPCClient-NioEventLoopGroup-6-5 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server fadb2fe0ba32:43811 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: fadb2fe0ba32/172.17.0.2:43811 Caused by: java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hbase.thirdparty.io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:336) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:339) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T04:25:51,374 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732422348811.ceffd2386e7c72b432c9dfad6355a885., hostname=fadb2fe0ba32,43811,1732422333705, seqNum=18 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1732422348811.ceffd2386e7c72b432c9dfad6355a885., hostname=fadb2fe0ba32,43811,1732422333705, seqNum=18, error=java.net.ConnectException: Call to address=fadb2fe0ba32:43811 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: fadb2fe0ba32/172.17.0.2:43811 2024-11-24T04:25:51,374 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732422348811.ceffd2386e7c72b432c9dfad6355a885., hostname=fadb2fe0ba32,43811,1732422333705, seqNum=18 is java.net.ConnectException: Connection refused 2024-11-24T04:25:51,374 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testReplayEditsAfterRegionMovedWithMultiCF,,1732422348811.ceffd2386e7c72b432c9dfad6355a885., hostname=fadb2fe0ba32,43811,1732422333705, seqNum=18 from cache 2024-11-24T04:25:51,375 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] ipc.FailedServers(52): Added failed server with address fadb2fe0ba32:43811 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: fadb2fe0ba32/172.17.0.2:43811 2024-11-24T04:25:51,446 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-24T04:25:51,448 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58563, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-24T04:25:51,449 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45209 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SplitWALCallable, pid=15 2024-11-24T04:25:51,469 INFO [RS_LOG_REPLAY_OPS-regionserver/fadb2fe0ba32:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(299): Splitting hdfs://localhost:44611/user/jenkins/test-data/33c38499-d2fe-b6f0-8d8b-5809759afd19/WALs/fadb2fe0ba32,43811,1732422333705-splitting/fadb2fe0ba32%2C43811%2C1732422333705.1732422335221, size=1.4 K (1407bytes) 2024-11-24T04:25:51,469 INFO [RS_LOG_REPLAY_OPS-regionserver/fadb2fe0ba32:0-0 {event_type=RS_LOG_REPLAY, pid=15}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:44611/user/jenkins/test-data/33c38499-d2fe-b6f0-8d8b-5809759afd19/WALs/fadb2fe0ba32,43811,1732422333705-splitting/fadb2fe0ba32%2C43811%2C1732422333705.1732422335221 2024-11-24T04:25:51,470 INFO [RS_LOG_REPLAY_OPS-regionserver/fadb2fe0ba32:0-0 {event_type=RS_LOG_REPLAY, pid=15}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:44611/user/jenkins/test-data/33c38499-d2fe-b6f0-8d8b-5809759afd19/WALs/fadb2fe0ba32,43811,1732422333705-splitting/fadb2fe0ba32%2C43811%2C1732422333705.1732422335221 after 1ms 2024-11-24T04:25:51,473 DEBUG [RS_LOG_REPLAY_OPS-regionserver/fadb2fe0ba32:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:44611/user/jenkins/test-data/33c38499-d2fe-b6f0-8d8b-5809759afd19/WALs/fadb2fe0ba32,43811,1732422333705-splitting/fadb2fe0ba32%2C43811%2C1732422333705.1732422335221: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-24T04:25:51,473 INFO [RS_LOG_REPLAY_OPS-regionserver/fadb2fe0ba32:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(310): Open hdfs://localhost:44611/user/jenkins/test-data/33c38499-d2fe-b6f0-8d8b-5809759afd19/WALs/fadb2fe0ba32,43811,1732422333705-splitting/fadb2fe0ba32%2C43811%2C1732422333705.1732422335221 took 4ms 2024-11-24T04:25:51,483 DEBUG [RS_LOG_REPLAY_OPS-regionserver/fadb2fe0ba32:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(352): Last flushed sequenceid for ceffd2386e7c72b432c9dfad6355a885: last_flushed_sequence_id: 12 store_sequence_id { family_name: "cf1" sequence_id: 12 } store_sequence_id { family_name: "cf2" sequence_id: 12 } 2024-11-24T04:25:51,483 DEBUG [RS_LOG_REPLAY_OPS-regionserver/fadb2fe0ba32:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:44611/user/jenkins/test-data/33c38499-d2fe-b6f0-8d8b-5809759afd19/WALs/fadb2fe0ba32,43811,1732422333705-splitting/fadb2fe0ba32%2C43811%2C1732422333705.1732422335221 so closing down 2024-11-24T04:25:51,484 DEBUG [RS_LOG_REPLAY_OPS-regionserver/fadb2fe0ba32:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-24T04:25:51,484 INFO [RS_LOG_REPLAY_OPS-regionserver/fadb2fe0ba32:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.OutputSink(145): 3 split writer threads finished 2024-11-24T04:25:51,484 INFO [RS_LOG_REPLAY_OPS-regionserver/fadb2fe0ba32:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(425): Processed 6 edits across 0 Regions in 10 ms; skipped=6; WAL=hdfs://localhost:44611/user/jenkins/test-data/33c38499-d2fe-b6f0-8d8b-5809759afd19/WALs/fadb2fe0ba32,43811,1732422333705-splitting/fadb2fe0ba32%2C43811%2C1732422333705.1732422335221, size=1.4 K, length=1407, corrupted=false, cancelled=false 2024-11-24T04:25:51,484 DEBUG [RS_LOG_REPLAY_OPS-regionserver/fadb2fe0ba32:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(428): Completed split of hdfs://localhost:44611/user/jenkins/test-data/33c38499-d2fe-b6f0-8d8b-5809759afd19/WALs/fadb2fe0ba32,43811,1732422333705-splitting/fadb2fe0ba32%2C43811%2C1732422333705.1732422335221, journal: Splitting hdfs://localhost:44611/user/jenkins/test-data/33c38499-d2fe-b6f0-8d8b-5809759afd19/WALs/fadb2fe0ba32,43811,1732422333705-splitting/fadb2fe0ba32%2C43811%2C1732422333705.1732422335221, size=1.4 K (1407bytes) at 1732422351469Finishing writing output for hdfs://localhost:44611/user/jenkins/test-data/33c38499-d2fe-b6f0-8d8b-5809759afd19/WALs/fadb2fe0ba32,43811,1732422333705-splitting/fadb2fe0ba32%2C43811%2C1732422333705.1732422335221 so closing down at 1732422351484 (+15 ms)3 split writer threads finished at 1732422351484Processed 6 edits across 0 Regions in 10 ms; skipped=6; WAL=hdfs://localhost:44611/user/jenkins/test-data/33c38499-d2fe-b6f0-8d8b-5809759afd19/WALs/fadb2fe0ba32,43811,1732422333705-splitting/fadb2fe0ba32%2C43811%2C1732422333705.1732422335221, size=1.4 K, length=1407, corrupted=false, cancelled=false at 1732422351484 2024-11-24T04:25:51,484 DEBUG [RS_LOG_REPLAY_OPS-regionserver/fadb2fe0ba32:0-0 {event_type=RS_LOG_REPLAY, pid=15}] regionserver.SplitLogWorker(218): Done splitting WAL hdfs://localhost:44611/user/jenkins/test-data/33c38499-d2fe-b6f0-8d8b-5809759afd19/WALs/fadb2fe0ba32,43811,1732422333705-splitting/fadb2fe0ba32%2C43811%2C1732422333705.1732422335221 2024-11-24T04:25:51,485 DEBUG [RS_LOG_REPLAY_OPS-regionserver/fadb2fe0ba32:0-0 {event_type=RS_LOG_REPLAY, pid=15}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=15 2024-11-24T04:25:51,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] master.HMaster(4169): Remote procedure done, pid=15 2024-11-24T04:25:51,492 INFO [PEWorker-4 {}] wal.WALSplitUtil(143): Moved hdfs://localhost:44611/user/jenkins/test-data/33c38499-d2fe-b6f0-8d8b-5809759afd19/WALs/fadb2fe0ba32,43811,1732422333705-splitting/fadb2fe0ba32%2C43811%2C1732422333705.1732422335221 to hdfs://localhost:44611/user/jenkins/test-data/33c38499-d2fe-b6f0-8d8b-5809759afd19/oldWALs 2024-11-24T04:25:51,495 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=15, resume processing ppid=14 2024-11-24T04:25:51,495 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, ppid=14, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure fadb2fe0ba32%2C43811%2C1732422333705.1732422335221, worker=fadb2fe0ba32,45209,1732422333751 in 205 msec 2024-11-24T04:25:51,497 DEBUG [PEWorker-5 {}] master.SplitWALManager(172): Release split WAL worker=fadb2fe0ba32,45209,1732422333751 2024-11-24T04:25:51,500 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-11-24T04:25:51,500 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; SplitWALProcedure fadb2fe0ba32%2C43811%2C1732422333705.1732422335221, worker=fadb2fe0ba32,45209,1732422333751 in 218 msec 2024-11-24T04:25:51,501 INFO [PEWorker-1 {}] master.SplitLogManager(171): hdfs://localhost:44611/user/jenkins/test-data/33c38499-d2fe-b6f0-8d8b-5809759afd19/WALs/fadb2fe0ba32,43811,1732422333705-splitting dir is empty, no logs to split. 2024-11-24T04:25:51,501 INFO [PEWorker-1 {}] master.SplitWALManager(105): fadb2fe0ba32,43811,1732422333705 WAL count=0, meta=false 2024-11-24T04:25:51,501 DEBUG [PEWorker-1 {}] procedure.ServerCrashProcedure(329): Check if fadb2fe0ba32,43811,1732422333705 WAL splitting is done? wals=0, meta=false 2024-11-24T04:25:51,504 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(321): Remove WAL directory for fadb2fe0ba32,43811,1732422333705 failed, ignore...File hdfs://localhost:44611/user/jenkins/test-data/33c38499-d2fe-b6f0-8d8b-5809759afd19/WALs/fadb2fe0ba32,43811,1732422333705-splitting does not exist. 2024-11-24T04:25:51,506 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=ceffd2386e7c72b432c9dfad6355a885, ASSIGN}] 2024-11-24T04:25:51,508 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=ceffd2386e7c72b432c9dfad6355a885, ASSIGN 2024-11-24T04:25:51,509 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=ceffd2386e7c72b432c9dfad6355a885, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-24T04:25:51,659 DEBUG [fadb2fe0ba32:33969 {}] balancer.BalancerClusterState(204): Hosts are {fadb2fe0ba32=0} racks are {/default-rack=0} 2024-11-24T04:25:51,660 DEBUG [fadb2fe0ba32:33969 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-24T04:25:51,660 DEBUG [fadb2fe0ba32:33969 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-24T04:25:51,660 DEBUG [fadb2fe0ba32:33969 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-24T04:25:51,660 DEBUG [fadb2fe0ba32:33969 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-24T04:25:51,660 INFO [fadb2fe0ba32:33969 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-24T04:25:51,660 INFO [fadb2fe0ba32:33969 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-24T04:25:51,660 DEBUG [fadb2fe0ba32:33969 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-24T04:25:51,660 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=ceffd2386e7c72b432c9dfad6355a885, regionState=OPENING, regionLocation=fadb2fe0ba32,45209,1732422333751 2024-11-24T04:25:51,663 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=ceffd2386e7c72b432c9dfad6355a885, ASSIGN because future has completed 2024-11-24T04:25:51,664 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=17, ppid=16, state=RUNNABLE, hasLock=false; OpenRegionProcedure ceffd2386e7c72b432c9dfad6355a885, server=fadb2fe0ba32,45209,1732422333751}] 2024-11-24T04:25:51,691 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1732422348811.ceffd2386e7c72b432c9dfad6355a885., hostname=fadb2fe0ba32,43811,1732422333705, seqNum=18] 2024-11-24T04:25:51,692 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] ipc.AbstractRpcClient(357): Not trying to connect to fadb2fe0ba32:43811 this server is in the failed servers list 2024-11-24T04:25:51,692 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732422348811.ceffd2386e7c72b432c9dfad6355a885., hostname=fadb2fe0ba32,43811,1732422333705, seqNum=18 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1732422348811.ceffd2386e7c72b432c9dfad6355a885., hostname=fadb2fe0ba32,43811,1732422333705, seqNum=18, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=fadb2fe0ba32:43811 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: fadb2fe0ba32:43811 2024-11-24T04:25:51,692 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732422348811.ceffd2386e7c72b432c9dfad6355a885., hostname=fadb2fe0ba32,43811,1732422333705, seqNum=18 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: fadb2fe0ba32:43811 2024-11-24T04:25:51,693 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testReplayEditsAfterRegionMovedWithMultiCF,,1732422348811.ceffd2386e7c72b432c9dfad6355a885., hostname=fadb2fe0ba32,43811,1732422333705, seqNum=18 from cache 2024-11-24T04:25:51,821 INFO [RS_OPEN_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1732422348811.ceffd2386e7c72b432c9dfad6355a885. 2024-11-24T04:25:51,821 DEBUG [RS_OPEN_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(7752): Opening region: {ENCODED => ceffd2386e7c72b432c9dfad6355a885, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1732422348811.ceffd2386e7c72b432c9dfad6355a885.', STARTKEY => '', ENDKEY => ''} 2024-11-24T04:25:51,821 DEBUG [RS_OPEN_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF ceffd2386e7c72b432c9dfad6355a885 2024-11-24T04:25:51,821 DEBUG [RS_OPEN_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1732422348811.ceffd2386e7c72b432c9dfad6355a885.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T04:25:51,821 DEBUG [RS_OPEN_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(7794): checking encryption for ceffd2386e7c72b432c9dfad6355a885 2024-11-24T04:25:51,822 DEBUG [RS_OPEN_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(7797): checking classloading for ceffd2386e7c72b432c9dfad6355a885 2024-11-24T04:25:51,823 INFO [StoreOpener-ceffd2386e7c72b432c9dfad6355a885-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region ceffd2386e7c72b432c9dfad6355a885 2024-11-24T04:25:51,824 INFO [StoreOpener-ceffd2386e7c72b432c9dfad6355a885-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ceffd2386e7c72b432c9dfad6355a885 columnFamilyName cf1 2024-11-24T04:25:51,824 DEBUG [StoreOpener-ceffd2386e7c72b432c9dfad6355a885-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:25:51,832 DEBUG [StoreOpener-ceffd2386e7c72b432c9dfad6355a885-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:44611/user/jenkins/test-data/33c38499-d2fe-b6f0-8d8b-5809759afd19/data/default/testReplayEditsAfterRegionMovedWithMultiCF/ceffd2386e7c72b432c9dfad6355a885/cf1/8e879a9777d543179208ddd4216bbc74 2024-11-24T04:25:51,832 INFO [StoreOpener-ceffd2386e7c72b432c9dfad6355a885-1 {}] regionserver.HStore(327): Store=ceffd2386e7c72b432c9dfad6355a885/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:25:51,832 INFO [StoreOpener-ceffd2386e7c72b432c9dfad6355a885-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region ceffd2386e7c72b432c9dfad6355a885 2024-11-24T04:25:51,833 INFO [StoreOpener-ceffd2386e7c72b432c9dfad6355a885-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ceffd2386e7c72b432c9dfad6355a885 columnFamilyName cf2 2024-11-24T04:25:51,833 DEBUG [StoreOpener-ceffd2386e7c72b432c9dfad6355a885-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:25:51,840 DEBUG [StoreOpener-ceffd2386e7c72b432c9dfad6355a885-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:44611/user/jenkins/test-data/33c38499-d2fe-b6f0-8d8b-5809759afd19/data/default/testReplayEditsAfterRegionMovedWithMultiCF/ceffd2386e7c72b432c9dfad6355a885/cf2/5f893655ca114f208aa3ea0462443f32 2024-11-24T04:25:51,840 INFO [StoreOpener-ceffd2386e7c72b432c9dfad6355a885-1 {}] regionserver.HStore(327): Store=ceffd2386e7c72b432c9dfad6355a885/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:25:51,840 DEBUG [RS_OPEN_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1038): replaying wal for ceffd2386e7c72b432c9dfad6355a885 2024-11-24T04:25:51,841 DEBUG [RS_OPEN_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44611/user/jenkins/test-data/33c38499-d2fe-b6f0-8d8b-5809759afd19/data/default/testReplayEditsAfterRegionMovedWithMultiCF/ceffd2386e7c72b432c9dfad6355a885 2024-11-24T04:25:51,843 DEBUG [RS_OPEN_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44611/user/jenkins/test-data/33c38499-d2fe-b6f0-8d8b-5809759afd19/data/default/testReplayEditsAfterRegionMovedWithMultiCF/ceffd2386e7c72b432c9dfad6355a885 2024-11-24T04:25:51,844 DEBUG [RS_OPEN_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1048): stopping wal replay for ceffd2386e7c72b432c9dfad6355a885 2024-11-24T04:25:51,844 DEBUG [RS_OPEN_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1060): Cleaning up temporary data for ceffd2386e7c72b432c9dfad6355a885 2024-11-24T04:25:51,845 DEBUG [RS_OPEN_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-11-24T04:25:51,846 DEBUG [RS_OPEN_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1093): writing seq id for ceffd2386e7c72b432c9dfad6355a885 2024-11-24T04:25:51,847 INFO [RS_OPEN_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1114): Opened ceffd2386e7c72b432c9dfad6355a885; next sequenceid=18; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72701383, jitterRate=0.0833350270986557}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-11-24T04:25:51,848 DEBUG [RS_OPEN_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1122): Running coprocessor post-open hooks for ceffd2386e7c72b432c9dfad6355a885 2024-11-24T04:25:51,849 DEBUG [RS_OPEN_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1006): Region open journal for ceffd2386e7c72b432c9dfad6355a885: Running coprocessor pre-open hook at 1732422351822Writing region info on filesystem at 1732422351822Initializing all the Stores at 1732422351823 (+1 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732422351823Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732422351823Cleaning up temporary data from old regions at 1732422351844 (+21 ms)Running coprocessor post-open hooks at 1732422351848 (+4 ms)Region opened successfully at 1732422351849 (+1 ms) 2024-11-24T04:25:51,851 INFO [RS_OPEN_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1732422348811.ceffd2386e7c72b432c9dfad6355a885., pid=17, masterSystemTime=1732422351816 2024-11-24T04:25:51,853 DEBUG [RS_OPEN_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1732422348811.ceffd2386e7c72b432c9dfad6355a885. 2024-11-24T04:25:51,854 INFO [RS_OPEN_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1732422348811.ceffd2386e7c72b432c9dfad6355a885. 2024-11-24T04:25:51,854 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=ceffd2386e7c72b432c9dfad6355a885, regionState=OPEN, openSeqNum=18, regionLocation=fadb2fe0ba32,45209,1732422333751 2024-11-24T04:25:51,857 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=17, ppid=16, state=RUNNABLE, hasLock=false; OpenRegionProcedure ceffd2386e7c72b432c9dfad6355a885, server=fadb2fe0ba32,45209,1732422333751 because future has completed 2024-11-24T04:25:51,868 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=17, resume processing ppid=16 2024-11-24T04:25:51,868 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, ppid=16, state=SUCCESS, hasLock=false; OpenRegionProcedure ceffd2386e7c72b432c9dfad6355a885, server=fadb2fe0ba32,45209,1732422333751 in 195 msec 2024-11-24T04:25:51,872 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=13 2024-11-24T04:25:51,872 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=13, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=ceffd2386e7c72b432c9dfad6355a885, ASSIGN in 362 msec 2024-11-24T04:25:51,872 INFO [PEWorker-4 {}] procedure.ServerCrashProcedure(291): removed crashed server fadb2fe0ba32,43811,1732422333705 after splitting done 2024-11-24T04:25:51,876 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; ServerCrashProcedure fadb2fe0ba32,43811,1732422333705, splitWal=true, meta=false in 608 msec 2024-11-24T04:25:52,201 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1732422348811.ceffd2386e7c72b432c9dfad6355a885., hostname=fadb2fe0ba32,45209,1732422333751, seqNum=18] 2024-11-24T04:25:52,201 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T04:25:52,203 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48966, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T04:25:52,222 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayValueCompression#testReplayEditsAfterRegionMovedWithMultiCF Thread=411 (was 415), OpenFileDescriptor=1048 (was 1022) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=196 (was 196), ProcessCount=11 (was 11), AvailableMemoryMB=6158 (was 6194) 2024-11-24T04:25:52,225 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1048 is superior to 1024 2024-11-24T04:25:52,244 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayValueCompression#testReplayEditsAfterPartialFlush Thread=411, OpenFileDescriptor=1048, MaxFileDescriptor=1048576, SystemLoadAverage=196, ProcessCount=11, AvailableMemoryMB=6158 2024-11-24T04:25:52,244 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1048 is superior to 1024 2024-11-24T04:25:52,260 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T04:25:52,262 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T04:25:52,263 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-24T04:25:52,265 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-43183938, suffix=, logDir=hdfs://localhost:44611/hbase/WALs/hregion-43183938, archiveDir=hdfs://localhost:44611/hbase/oldWALs, maxLogs=32 2024-11-24T04:25:52,282 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-43183938/hregion-43183938.1732422352266, exclude list is [], retry=0 2024-11-24T04:25:52,285 DEBUG [AsyncFSWAL-20-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34457,DS-20bb5930-23a3-441d-bdb2-015c0d56b2ec,DISK] 2024-11-24T04:25:52,285 DEBUG [AsyncFSWAL-20-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44781,DS-54254ac8-0fae-43f7-ab06-53faa0e05dd7,DISK] 2024-11-24T04:25:52,287 DEBUG [AsyncFSWAL-20-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36069,DS-11c8f594-4fc2-47e4-b222-092ec50eb991,DISK] 2024-11-24T04:25:52,297 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-43183938/hregion-43183938.1732422352266 2024-11-24T04:25:52,298 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33347:33347),(127.0.0.1/127.0.0.1:35691:35691),(127.0.0.1/127.0.0.1:41009:41009)] 2024-11-24T04:25:52,298 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => b199289b7beb5e483741afe3874ae427, NAME => 'testReplayEditsWrittenViaHRegion,,1732422352261.b199289b7beb5e483741afe3874ae427.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenViaHRegion', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44611/hbase 2024-11-24T04:25:52,315 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36069 is added to blk_1073741893_1071 (size=67) 2024-11-24T04:25:52,315 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34457 is added to blk_1073741893_1071 (size=67) 2024-11-24T04:25:52,316 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44781 is added to blk_1073741893_1071 (size=67) 2024-11-24T04:25:52,317 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1732422352261.b199289b7beb5e483741afe3874ae427.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T04:25:52,319 INFO [StoreOpener-b199289b7beb5e483741afe3874ae427-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region b199289b7beb5e483741afe3874ae427 2024-11-24T04:25:52,321 INFO [StoreOpener-b199289b7beb5e483741afe3874ae427-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b199289b7beb5e483741afe3874ae427 columnFamilyName a 2024-11-24T04:25:52,322 DEBUG [StoreOpener-b199289b7beb5e483741afe3874ae427-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:25:52,322 INFO [StoreOpener-b199289b7beb5e483741afe3874ae427-1 {}] regionserver.HStore(327): Store=b199289b7beb5e483741afe3874ae427/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:25:52,322 INFO [StoreOpener-b199289b7beb5e483741afe3874ae427-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region b199289b7beb5e483741afe3874ae427 2024-11-24T04:25:52,324 INFO [StoreOpener-b199289b7beb5e483741afe3874ae427-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b199289b7beb5e483741afe3874ae427 columnFamilyName b 2024-11-24T04:25:52,324 DEBUG [StoreOpener-b199289b7beb5e483741afe3874ae427-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:25:52,325 INFO [StoreOpener-b199289b7beb5e483741afe3874ae427-1 {}] regionserver.HStore(327): Store=b199289b7beb5e483741afe3874ae427/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:25:52,325 INFO [StoreOpener-b199289b7beb5e483741afe3874ae427-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region b199289b7beb5e483741afe3874ae427 2024-11-24T04:25:52,326 INFO [StoreOpener-b199289b7beb5e483741afe3874ae427-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b199289b7beb5e483741afe3874ae427 columnFamilyName c 2024-11-24T04:25:52,326 DEBUG [StoreOpener-b199289b7beb5e483741afe3874ae427-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:25:52,327 INFO [StoreOpener-b199289b7beb5e483741afe3874ae427-1 {}] regionserver.HStore(327): Store=b199289b7beb5e483741afe3874ae427/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:25:52,327 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for b199289b7beb5e483741afe3874ae427 2024-11-24T04:25:52,328 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44611/hbase/data/default/testReplayEditsWrittenViaHRegion/b199289b7beb5e483741afe3874ae427 2024-11-24T04:25:52,328 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44611/hbase/data/default/testReplayEditsWrittenViaHRegion/b199289b7beb5e483741afe3874ae427 2024-11-24T04:25:52,329 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for b199289b7beb5e483741afe3874ae427 2024-11-24T04:25:52,329 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for b199289b7beb5e483741afe3874ae427 2024-11-24T04:25:52,330 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-24T04:25:52,331 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for b199289b7beb5e483741afe3874ae427 2024-11-24T04:25:52,334 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44611/hbase/data/default/testReplayEditsWrittenViaHRegion/b199289b7beb5e483741afe3874ae427/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T04:25:52,334 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened b199289b7beb5e483741afe3874ae427; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63665880, jitterRate=-0.05130445957183838}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-24T04:25:52,335 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for b199289b7beb5e483741afe3874ae427: Writing region info on filesystem at 1732422352317Initializing all the Stores at 1732422352319 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732422352319Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732422352319Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732422352319Cleaning up temporary data from old regions at 1732422352329 (+10 ms)Region opened successfully at 1732422352335 (+6 ms) 2024-11-24T04:25:52,335 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing b199289b7beb5e483741afe3874ae427, disabling compactions & flushes 2024-11-24T04:25:52,335 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1732422352261.b199289b7beb5e483741afe3874ae427. 2024-11-24T04:25:52,335 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1732422352261.b199289b7beb5e483741afe3874ae427. 2024-11-24T04:25:52,335 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1732422352261.b199289b7beb5e483741afe3874ae427. after waiting 0 ms 2024-11-24T04:25:52,335 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1732422352261.b199289b7beb5e483741afe3874ae427. 2024-11-24T04:25:52,336 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1732422352261.b199289b7beb5e483741afe3874ae427. 2024-11-24T04:25:52,336 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for b199289b7beb5e483741afe3874ae427: Waiting for close lock at 1732422352335Disabling compacts and flushes for region at 1732422352335Disabling writes for close at 1732422352335Writing region close event to WAL at 1732422352336 (+1 ms)Closed at 1732422352336 2024-11-24T04:25:52,339 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44781 is added to blk_1073741892_1070 (size=95) 2024-11-24T04:25:52,339 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36069 is added to blk_1073741892_1070 (size=95) 2024-11-24T04:25:52,339 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34457 is added to blk_1073741892_1070 (size=95) 2024-11-24T04:25:52,341 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-24T04:25:52,341 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-43183938:(num 1732422352266) 2024-11-24T04:25:52,341 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-24T04:25:52,343 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:44611/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732422352259, archiveDir=hdfs://localhost:44611/hbase/oldWALs, maxLogs=32 2024-11-24T04:25:52,361 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732422352259/wal.1732422352344, exclude list is [], retry=0 2024-11-24T04:25:52,364 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36069,DS-11c8f594-4fc2-47e4-b222-092ec50eb991,DISK] 2024-11-24T04:25:52,365 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44781,DS-54254ac8-0fae-43f7-ab06-53faa0e05dd7,DISK] 2024-11-24T04:25:52,365 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34457,DS-20bb5930-23a3-441d-bdb2-015c0d56b2ec,DISK] 2024-11-24T04:25:52,367 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732422352259/wal.1732422352344 2024-11-24T04:25:52,368 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41009:41009),(127.0.0.1/127.0.0.1:35691:35691),(127.0.0.1/127.0.0.1:33347:33347)] 2024-11-24T04:25:52,368 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => b199289b7beb5e483741afe3874ae427, NAME => 'testReplayEditsWrittenViaHRegion,,1732422352261.b199289b7beb5e483741afe3874ae427.', STARTKEY => '', ENDKEY => ''} 2024-11-24T04:25:52,368 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1732422352261.b199289b7beb5e483741afe3874ae427.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T04:25:52,368 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for b199289b7beb5e483741afe3874ae427 2024-11-24T04:25:52,368 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for b199289b7beb5e483741afe3874ae427 2024-11-24T04:25:52,370 INFO [StoreOpener-b199289b7beb5e483741afe3874ae427-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region b199289b7beb5e483741afe3874ae427 2024-11-24T04:25:52,371 INFO [StoreOpener-b199289b7beb5e483741afe3874ae427-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b199289b7beb5e483741afe3874ae427 columnFamilyName a 2024-11-24T04:25:52,371 DEBUG [StoreOpener-b199289b7beb5e483741afe3874ae427-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:25:52,371 INFO [StoreOpener-b199289b7beb5e483741afe3874ae427-1 {}] regionserver.HStore(327): Store=b199289b7beb5e483741afe3874ae427/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:25:52,372 INFO [StoreOpener-b199289b7beb5e483741afe3874ae427-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region b199289b7beb5e483741afe3874ae427 2024-11-24T04:25:52,373 INFO [StoreOpener-b199289b7beb5e483741afe3874ae427-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b199289b7beb5e483741afe3874ae427 columnFamilyName b 2024-11-24T04:25:52,373 DEBUG [StoreOpener-b199289b7beb5e483741afe3874ae427-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:25:52,373 INFO [StoreOpener-b199289b7beb5e483741afe3874ae427-1 {}] regionserver.HStore(327): Store=b199289b7beb5e483741afe3874ae427/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:25:52,373 INFO [StoreOpener-b199289b7beb5e483741afe3874ae427-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region b199289b7beb5e483741afe3874ae427 2024-11-24T04:25:52,374 INFO [StoreOpener-b199289b7beb5e483741afe3874ae427-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b199289b7beb5e483741afe3874ae427 columnFamilyName c 2024-11-24T04:25:52,374 DEBUG [StoreOpener-b199289b7beb5e483741afe3874ae427-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:25:52,375 INFO [StoreOpener-b199289b7beb5e483741afe3874ae427-1 {}] regionserver.HStore(327): Store=b199289b7beb5e483741afe3874ae427/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:25:52,375 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for b199289b7beb5e483741afe3874ae427 2024-11-24T04:25:52,376 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44611/hbase/data/default/testReplayEditsWrittenViaHRegion/b199289b7beb5e483741afe3874ae427 2024-11-24T04:25:52,377 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44611/hbase/data/default/testReplayEditsWrittenViaHRegion/b199289b7beb5e483741afe3874ae427 2024-11-24T04:25:52,378 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for b199289b7beb5e483741afe3874ae427 2024-11-24T04:25:52,378 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for b199289b7beb5e483741afe3874ae427 2024-11-24T04:25:52,379 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-24T04:25:52,380 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for b199289b7beb5e483741afe3874ae427 2024-11-24T04:25:52,381 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened b199289b7beb5e483741afe3874ae427; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69274538, jitterRate=0.03227105736732483}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-24T04:25:52,382 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for b199289b7beb5e483741afe3874ae427: Writing region info on filesystem at 1732422352368Initializing all the Stores at 1732422352369 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732422352369Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732422352370 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732422352370Cleaning up temporary data from old regions at 1732422352378 (+8 ms)Region opened successfully at 1732422352382 (+4 ms) 2024-11-24T04:25:52,421 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing b199289b7beb5e483741afe3874ae427 3/3 column families, dataSize=2.55 KB heapSize=5.44 KB 2024-11-24T04:25:52,445 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44611/hbase/data/default/testReplayEditsWrittenViaHRegion/b199289b7beb5e483741afe3874ae427/.tmp/a/b3996d72c91f43c5ba93d2367b4d417c is 91, key is testReplayEditsWrittenViaHRegion/a:x0/1732422352383/Put/seqid=0 2024-11-24T04:25:52,453 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44781 is added to blk_1073741895_1073 (size=5958) 2024-11-24T04:25:52,457 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36069 is added to blk_1073741895_1073 (size=5958) 2024-11-24T04:25:52,458 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34457 is added to blk_1073741895_1073 (size=5958) 2024-11-24T04:25:52,458 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:44611/hbase/data/default/testReplayEditsWrittenViaHRegion/b199289b7beb5e483741afe3874ae427/.tmp/a/b3996d72c91f43c5ba93d2367b4d417c 2024-11-24T04:25:52,484 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44611/hbase/data/default/testReplayEditsWrittenViaHRegion/b199289b7beb5e483741afe3874ae427/.tmp/b/eefe4188586f4d668f910defc1eaf278 is 91, key is testReplayEditsWrittenViaHRegion/b:x0/1732422352395/Put/seqid=0 2024-11-24T04:25:52,491 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34457 is added to blk_1073741896_1074 (size=5958) 2024-11-24T04:25:52,492 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36069 is added to blk_1073741896_1074 (size=5958) 2024-11-24T04:25:52,492 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44781 is added to blk_1073741896_1074 (size=5958) 2024-11-24T04:25:52,494 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:44611/hbase/data/default/testReplayEditsWrittenViaHRegion/b199289b7beb5e483741afe3874ae427/.tmp/b/eefe4188586f4d668f910defc1eaf278 2024-11-24T04:25:52,522 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44611/hbase/data/default/testReplayEditsWrittenViaHRegion/b199289b7beb5e483741afe3874ae427/.tmp/c/f9b8ef2cf86542acbc0bc0d91f0d5792 is 91, key is testReplayEditsWrittenViaHRegion/c:x0/1732422352408/Put/seqid=0 2024-11-24T04:25:52,530 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44781 is added to blk_1073741897_1075 (size=5958) 2024-11-24T04:25:52,530 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36069 is added to blk_1073741897_1075 (size=5958) 2024-11-24T04:25:52,530 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34457 is added to blk_1073741897_1075 (size=5958) 2024-11-24T04:25:52,533 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:44611/hbase/data/default/testReplayEditsWrittenViaHRegion/b199289b7beb5e483741afe3874ae427/.tmp/c/f9b8ef2cf86542acbc0bc0d91f0d5792 2024-11-24T04:25:52,540 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44611/hbase/data/default/testReplayEditsWrittenViaHRegion/b199289b7beb5e483741afe3874ae427/.tmp/a/b3996d72c91f43c5ba93d2367b4d417c as hdfs://localhost:44611/hbase/data/default/testReplayEditsWrittenViaHRegion/b199289b7beb5e483741afe3874ae427/a/b3996d72c91f43c5ba93d2367b4d417c 2024-11-24T04:25:52,546 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44611/hbase/data/default/testReplayEditsWrittenViaHRegion/b199289b7beb5e483741afe3874ae427/a/b3996d72c91f43c5ba93d2367b4d417c, entries=10, sequenceid=33, filesize=5.8 K 2024-11-24T04:25:52,548 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44611/hbase/data/default/testReplayEditsWrittenViaHRegion/b199289b7beb5e483741afe3874ae427/.tmp/b/eefe4188586f4d668f910defc1eaf278 as hdfs://localhost:44611/hbase/data/default/testReplayEditsWrittenViaHRegion/b199289b7beb5e483741afe3874ae427/b/eefe4188586f4d668f910defc1eaf278 2024-11-24T04:25:52,556 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44611/hbase/data/default/testReplayEditsWrittenViaHRegion/b199289b7beb5e483741afe3874ae427/b/eefe4188586f4d668f910defc1eaf278, entries=10, sequenceid=33, filesize=5.8 K 2024-11-24T04:25:52,557 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44611/hbase/data/default/testReplayEditsWrittenViaHRegion/b199289b7beb5e483741afe3874ae427/.tmp/c/f9b8ef2cf86542acbc0bc0d91f0d5792 as hdfs://localhost:44611/hbase/data/default/testReplayEditsWrittenViaHRegion/b199289b7beb5e483741afe3874ae427/c/f9b8ef2cf86542acbc0bc0d91f0d5792 2024-11-24T04:25:52,563 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44611/hbase/data/default/testReplayEditsWrittenViaHRegion/b199289b7beb5e483741afe3874ae427/c/f9b8ef2cf86542acbc0bc0d91f0d5792, entries=10, sequenceid=33, filesize=5.8 K 2024-11-24T04:25:52,564 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~2.55 KB/2610, heapSize ~5.39 KB/5520, currentSize=0 B/0 for b199289b7beb5e483741afe3874ae427 in 143ms, sequenceid=33, compaction requested=false 2024-11-24T04:25:52,565 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for b199289b7beb5e483741afe3874ae427: 2024-11-24T04:25:52,565 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing b199289b7beb5e483741afe3874ae427, disabling compactions & flushes 2024-11-24T04:25:52,565 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1732422352261.b199289b7beb5e483741afe3874ae427. 2024-11-24T04:25:52,565 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1732422352261.b199289b7beb5e483741afe3874ae427. 2024-11-24T04:25:52,565 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1732422352261.b199289b7beb5e483741afe3874ae427. after waiting 0 ms 2024-11-24T04:25:52,565 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1732422352261.b199289b7beb5e483741afe3874ae427. 2024-11-24T04:25:52,567 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1732422352261.b199289b7beb5e483741afe3874ae427. 2024-11-24T04:25:52,567 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for b199289b7beb5e483741afe3874ae427: Waiting for close lock at 1732422352565Disabling compacts and flushes for region at 1732422352565Disabling writes for close at 1732422352565Writing region close event to WAL at 1732422352567 (+2 ms)Closed at 1732422352567 2024-11-24T04:25:52,570 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34457 is added to blk_1073741894_1072 (size=3387) 2024-11-24T04:25:52,570 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36069 is added to blk_1073741894_1072 (size=3387) 2024-11-24T04:25:52,570 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44781 is added to blk_1073741894_1072 (size=3387) 2024-11-24T04:25:52,575 DEBUG [Time-limited test {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44611/hbase/data/default/testReplayEditsWrittenViaHRegion/b199289b7beb5e483741afe3874ae427/b/eefe4188586f4d668f910defc1eaf278 to hdfs://localhost:44611/hbase/archive/data/default/testReplayEditsWrittenViaHRegion/b199289b7beb5e483741afe3874ae427/b/eefe4188586f4d668f910defc1eaf278 2024-11-24T04:25:52,594 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:44611/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732422352259/wal.1732422352344, size=3.3 K (3387bytes) 2024-11-24T04:25:52,594 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:44611/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732422352259/wal.1732422352344 2024-11-24T04:25:52,595 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:44611/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732422352259/wal.1732422352344 after 1ms 2024-11-24T04:25:52,597 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:44611/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732422352259/wal.1732422352344: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-24T04:25:52,597 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:44611/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732422352259/wal.1732422352344 took 3ms 2024-11-24T04:25:52,599 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:44611/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732422352259/wal.1732422352344 so closing down 2024-11-24T04:25:52,599 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-24T04:25:52,600 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1732422352344.temp 2024-11-24T04:25:52,602 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:44611/hbase/data/default/testReplayEditsWrittenViaHRegion/b199289b7beb5e483741afe3874ae427/recovered.edits/0000000000000000003-wal.1732422352344.temp 2024-11-24T04:25:52,603 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-24T04:25:52,609 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34457 is added to blk_1073741898_1076 (size=2944) 2024-11-24T04:25:52,609 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36069 is added to blk_1073741898_1076 (size=2944) 2024-11-24T04:25:52,610 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44781 is added to blk_1073741898_1076 (size=2944) 2024-11-24T04:25:52,610 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:44611/hbase/data/default/testReplayEditsWrittenViaHRegion/b199289b7beb5e483741afe3874ae427/recovered.edits/0000000000000000003-wal.1732422352344.temp (wrote 30 edits, skipped 0 edits in 1 ms) 2024-11-24T04:25:52,611 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:44611/hbase/data/default/testReplayEditsWrittenViaHRegion/b199289b7beb5e483741afe3874ae427/recovered.edits/0000000000000000003-wal.1732422352344.temp to hdfs://localhost:44611/hbase/data/default/testReplayEditsWrittenViaHRegion/b199289b7beb5e483741afe3874ae427/recovered.edits/0000000000000000032 2024-11-24T04:25:52,612 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 32 edits across 1 Regions in 14 ms; skipped=2; WAL=hdfs://localhost:44611/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732422352259/wal.1732422352344, size=3.3 K, length=3387, corrupted=false, cancelled=false 2024-11-24T04:25:52,612 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:44611/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732422352259/wal.1732422352344, journal: Splitting hdfs://localhost:44611/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732422352259/wal.1732422352344, size=3.3 K (3387bytes) at 1732422352594Finishing writing output for hdfs://localhost:44611/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732422352259/wal.1732422352344 so closing down at 1732422352599 (+5 ms)Creating recovered edits writer path=hdfs://localhost:44611/hbase/data/default/testReplayEditsWrittenViaHRegion/b199289b7beb5e483741afe3874ae427/recovered.edits/0000000000000000003-wal.1732422352344.temp at 1732422352602 (+3 ms)3 split writer threads finished at 1732422352603 (+1 ms)Closed recovered edits writer path=hdfs://localhost:44611/hbase/data/default/testReplayEditsWrittenViaHRegion/b199289b7beb5e483741afe3874ae427/recovered.edits/0000000000000000003-wal.1732422352344.temp (wrote 30 edits, skipped 0 edits in 1 ms) at 1732422352610 (+7 ms)Rename recovered edits hdfs://localhost:44611/hbase/data/default/testReplayEditsWrittenViaHRegion/b199289b7beb5e483741afe3874ae427/recovered.edits/0000000000000000003-wal.1732422352344.temp to hdfs://localhost:44611/hbase/data/default/testReplayEditsWrittenViaHRegion/b199289b7beb5e483741afe3874ae427/recovered.edits/0000000000000000032 at 1732422352611 (+1 ms)Processed 32 edits across 1 Regions in 14 ms; skipped=2; WAL=hdfs://localhost:44611/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732422352259/wal.1732422352344, size=3.3 K, length=3387, corrupted=false, cancelled=false at 1732422352612 (+1 ms) 2024-11-24T04:25:52,613 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:44611/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732422352259/wal.1732422352344 to hdfs://localhost:44611/hbase/oldWALs/wal.1732422352344 2024-11-24T04:25:52,614 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:44611/hbase/data/default/testReplayEditsWrittenViaHRegion/b199289b7beb5e483741afe3874ae427/recovered.edits/0000000000000000032 2024-11-24T04:25:52,614 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-24T04:25:52,616 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:44611/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732422352259, archiveDir=hdfs://localhost:44611/hbase/oldWALs, maxLogs=32 2024-11-24T04:25:52,630 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732422352259/wal.1732422352616, exclude list is [], retry=0 2024-11-24T04:25:52,632 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44781,DS-54254ac8-0fae-43f7-ab06-53faa0e05dd7,DISK] 2024-11-24T04:25:52,632 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34457,DS-20bb5930-23a3-441d-bdb2-015c0d56b2ec,DISK] 2024-11-24T04:25:52,633 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36069,DS-11c8f594-4fc2-47e4-b222-092ec50eb991,DISK] 2024-11-24T04:25:52,634 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732422352259/wal.1732422352616 2024-11-24T04:25:52,635 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35691:35691),(127.0.0.1/127.0.0.1:33347:33347),(127.0.0.1/127.0.0.1:41009:41009)] 2024-11-24T04:25:52,635 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => b199289b7beb5e483741afe3874ae427, NAME => 'testReplayEditsWrittenViaHRegion,,1732422352261.b199289b7beb5e483741afe3874ae427.', STARTKEY => '', ENDKEY => ''} 2024-11-24T04:25:52,635 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1732422352261.b199289b7beb5e483741afe3874ae427.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T04:25:52,635 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for b199289b7beb5e483741afe3874ae427 2024-11-24T04:25:52,635 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for b199289b7beb5e483741afe3874ae427 2024-11-24T04:25:52,637 INFO [StoreOpener-b199289b7beb5e483741afe3874ae427-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region b199289b7beb5e483741afe3874ae427 2024-11-24T04:25:52,638 INFO [StoreOpener-b199289b7beb5e483741afe3874ae427-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b199289b7beb5e483741afe3874ae427 columnFamilyName a 2024-11-24T04:25:52,638 DEBUG [StoreOpener-b199289b7beb5e483741afe3874ae427-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:25:52,643 DEBUG [StoreOpener-b199289b7beb5e483741afe3874ae427-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:44611/hbase/data/default/testReplayEditsWrittenViaHRegion/b199289b7beb5e483741afe3874ae427/a/b3996d72c91f43c5ba93d2367b4d417c 2024-11-24T04:25:52,643 INFO [StoreOpener-b199289b7beb5e483741afe3874ae427-1 {}] regionserver.HStore(327): Store=b199289b7beb5e483741afe3874ae427/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:25:52,643 INFO [StoreOpener-b199289b7beb5e483741afe3874ae427-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region b199289b7beb5e483741afe3874ae427 2024-11-24T04:25:52,644 INFO [StoreOpener-b199289b7beb5e483741afe3874ae427-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b199289b7beb5e483741afe3874ae427 columnFamilyName b 2024-11-24T04:25:52,645 DEBUG [StoreOpener-b199289b7beb5e483741afe3874ae427-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:25:52,645 INFO [StoreOpener-b199289b7beb5e483741afe3874ae427-1 {}] regionserver.HStore(327): Store=b199289b7beb5e483741afe3874ae427/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:25:52,645 INFO [StoreOpener-b199289b7beb5e483741afe3874ae427-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region b199289b7beb5e483741afe3874ae427 2024-11-24T04:25:52,646 INFO [StoreOpener-b199289b7beb5e483741afe3874ae427-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b199289b7beb5e483741afe3874ae427 columnFamilyName c 2024-11-24T04:25:52,646 DEBUG [StoreOpener-b199289b7beb5e483741afe3874ae427-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:25:52,651 DEBUG [StoreOpener-b199289b7beb5e483741afe3874ae427-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:44611/hbase/data/default/testReplayEditsWrittenViaHRegion/b199289b7beb5e483741afe3874ae427/c/f9b8ef2cf86542acbc0bc0d91f0d5792 2024-11-24T04:25:52,651 INFO [StoreOpener-b199289b7beb5e483741afe3874ae427-1 {}] regionserver.HStore(327): Store=b199289b7beb5e483741afe3874ae427/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:25:52,651 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for b199289b7beb5e483741afe3874ae427 2024-11-24T04:25:52,652 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44611/hbase/data/default/testReplayEditsWrittenViaHRegion/b199289b7beb5e483741afe3874ae427 2024-11-24T04:25:52,654 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:44611/hbase/data/default/testReplayEditsWrittenViaHRegion/b199289b7beb5e483741afe3874ae427 2024-11-24T04:25:52,654 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:44611/hbase/data/default/testReplayEditsWrittenViaHRegion/b199289b7beb5e483741afe3874ae427/recovered.edits/0000000000000000032 2024-11-24T04:25:52,656 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:44611/hbase/data/default/testReplayEditsWrittenViaHRegion/b199289b7beb5e483741afe3874ae427/recovered.edits/0000000000000000032: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-24T04:25:52,658 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 10, skipped 20, firstSequenceIdInLog=3, maxSequenceIdInLog=32, path=hdfs://localhost:44611/hbase/data/default/testReplayEditsWrittenViaHRegion/b199289b7beb5e483741afe3874ae427/recovered.edits/0000000000000000032 2024-11-24T04:25:52,658 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing b199289b7beb5e483741afe3874ae427 3/3 column families, dataSize=870 B heapSize=2.31 KB 2024-11-24T04:25:52,674 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44611/hbase/data/default/testReplayEditsWrittenViaHRegion/b199289b7beb5e483741afe3874ae427/.tmp/b/a51c90c330b34e62bc27d582d8df17d8 is 91, key is testReplayEditsWrittenViaHRegion/b:x0/1732422352395/Put/seqid=0 2024-11-24T04:25:52,680 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34457 is added to blk_1073741900_1078 (size=5958) 2024-11-24T04:25:52,681 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44781 is added to blk_1073741900_1078 (size=5958) 2024-11-24T04:25:52,681 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36069 is added to blk_1073741900_1078 (size=5958) 2024-11-24T04:25:52,683 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=32 (bloomFilter=true), to=hdfs://localhost:44611/hbase/data/default/testReplayEditsWrittenViaHRegion/b199289b7beb5e483741afe3874ae427/.tmp/b/a51c90c330b34e62bc27d582d8df17d8 2024-11-24T04:25:52,688 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44611/hbase/data/default/testReplayEditsWrittenViaHRegion/b199289b7beb5e483741afe3874ae427/.tmp/b/a51c90c330b34e62bc27d582d8df17d8 as hdfs://localhost:44611/hbase/data/default/testReplayEditsWrittenViaHRegion/b199289b7beb5e483741afe3874ae427/b/a51c90c330b34e62bc27d582d8df17d8 2024-11-24T04:25:52,694 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44611/hbase/data/default/testReplayEditsWrittenViaHRegion/b199289b7beb5e483741afe3874ae427/b/a51c90c330b34e62bc27d582d8df17d8, entries=10, sequenceid=32, filesize=5.8 K 2024-11-24T04:25:52,694 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~870 B/870, heapSize ~1.80 KB/1840, currentSize=0 B/0 for b199289b7beb5e483741afe3874ae427 in 36ms, sequenceid=32, compaction requested=false; wal=null 2024-11-24T04:25:52,695 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:44611/hbase/data/default/testReplayEditsWrittenViaHRegion/b199289b7beb5e483741afe3874ae427/recovered.edits/0000000000000000032 2024-11-24T04:25:52,696 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for b199289b7beb5e483741afe3874ae427 2024-11-24T04:25:52,696 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for b199289b7beb5e483741afe3874ae427 2024-11-24T04:25:52,697 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-24T04:25:52,698 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for b199289b7beb5e483741afe3874ae427 2024-11-24T04:25:52,701 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44611/hbase/data/default/testReplayEditsWrittenViaHRegion/b199289b7beb5e483741afe3874ae427/recovered.edits/33.seqid, newMaxSeqId=33, maxSeqId=1 2024-11-24T04:25:52,702 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened b199289b7beb5e483741afe3874ae427; next sequenceid=34; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74680779, jitterRate=0.11283032596111298}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-24T04:25:52,703 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for b199289b7beb5e483741afe3874ae427: Writing region info on filesystem at 1732422352635Initializing all the Stores at 1732422352636 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732422352636Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732422352636Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732422352636Obtaining lock to block concurrent updates at 1732422352658 (+22 ms)Preparing flush snapshotting stores in b199289b7beb5e483741afe3874ae427 at 1732422352658Finished memstore snapshotting testReplayEditsWrittenViaHRegion,,1732422352261.b199289b7beb5e483741afe3874ae427., syncing WAL and waiting on mvcc, flushsize=dataSize=870, getHeapSize=2320, getOffHeapSize=0, getCellsCount=10 at 1732422352658Flushing stores of testReplayEditsWrittenViaHRegion,,1732422352261.b199289b7beb5e483741afe3874ae427. at 1732422352658Flushing b199289b7beb5e483741afe3874ae427/b: creating writer at 1732422352658Flushing b199289b7beb5e483741afe3874ae427/b: appending metadata at 1732422352674 (+16 ms)Flushing b199289b7beb5e483741afe3874ae427/b: closing flushed file at 1732422352674Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@68a57d7d: reopening flushed file at 1732422352688 (+14 ms)Finished flush of dataSize ~870 B/870, heapSize ~1.80 KB/1840, currentSize=0 B/0 for b199289b7beb5e483741afe3874ae427 in 36ms, sequenceid=32, compaction requested=false; wal=null at 1732422352694 (+6 ms)Cleaning up temporary data from old regions at 1732422352696 (+2 ms)Region opened successfully at 1732422352703 (+7 ms) 2024-11-24T04:25:52,728 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayValueCompression#testReplayEditsAfterPartialFlush Thread=420 (was 411) Potentially hanging thread: AsyncFSWAL-20-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1575425110_22 at /127.0.0.1:34130 [Waiting for operation #31] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-20-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1575425110_22 at /127.0.0.1:56570 [Receiving block BP-1091946017-172.17.0.2-1732422329789:blk_1073741899_1077] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1575425110_22 at /127.0.0.1:46870 [Waiting for operation #38] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-20-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1091946017-172.17.0.2-1732422329789:blk_1073741899_1077, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1091946017-172.17.0.2-1732422329789:blk_1073741899_1077, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1575425110_22 at /127.0.0.1:50168 [Waiting for operation #33] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1575425110_22 at /127.0.0.1:51172 [Receiving block BP-1091946017-172.17.0.2-1732422329789:blk_1073741899_1077] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1575425110_22 at /127.0.0.1:41058 [Receiving block BP-1091946017-172.17.0.2-1732422329789:blk_1073741899_1077] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1091946017-172.17.0.2-1732422329789:blk_1073741899_1077, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1124 (was 1048) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=196 (was 196), ProcessCount=11 (was 11), AvailableMemoryMB=6150 (was 6158) 2024-11-24T04:25:52,729 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1124 is superior to 1024 2024-11-24T04:25:52,741 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayValueCompression#testReplayEditsAfterAbortingFlush Thread=420, OpenFileDescriptor=1124, MaxFileDescriptor=1048576, SystemLoadAverage=196, ProcessCount=11, AvailableMemoryMB=6149 2024-11-24T04:25:52,741 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1124 is superior to 1024 2024-11-24T04:25:52,757 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T04:25:52,759 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T04:25:52,759 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-24T04:25:52,762 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-07185885, suffix=, logDir=hdfs://localhost:44611/hbase/WALs/hregion-07185885, archiveDir=hdfs://localhost:44611/hbase/oldWALs, maxLogs=32 2024-11-24T04:25:52,775 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-07185885/hregion-07185885.1732422352762, exclude list is [], retry=0 2024-11-24T04:25:52,778 DEBUG [AsyncFSWAL-22-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34457,DS-20bb5930-23a3-441d-bdb2-015c0d56b2ec,DISK] 2024-11-24T04:25:52,779 DEBUG [AsyncFSWAL-22-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36069,DS-11c8f594-4fc2-47e4-b222-092ec50eb991,DISK] 2024-11-24T04:25:52,780 DEBUG [AsyncFSWAL-22-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44781,DS-54254ac8-0fae-43f7-ab06-53faa0e05dd7,DISK] 2024-11-24T04:25:52,782 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-07185885/hregion-07185885.1732422352762 2024-11-24T04:25:52,784 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33347:33347),(127.0.0.1/127.0.0.1:41009:41009),(127.0.0.1/127.0.0.1:35691:35691)] 2024-11-24T04:25:52,785 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 478fe28262f058acb865d9d56ec12253, NAME => 'testReplayEditsAfterAbortingFlush,,1732422352758.478fe28262f058acb865d9d56ec12253.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsAfterAbortingFlush', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44611/hbase 2024-11-24T04:25:52,800 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44781 is added to blk_1073741902_1080 (size=68) 2024-11-24T04:25:52,800 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36069 is added to blk_1073741902_1080 (size=68) 2024-11-24T04:25:52,800 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34457 is added to blk_1073741902_1080 (size=68) 2024-11-24T04:25:52,802 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterAbortingFlush,,1732422352758.478fe28262f058acb865d9d56ec12253.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T04:25:52,804 INFO [StoreOpener-478fe28262f058acb865d9d56ec12253-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 478fe28262f058acb865d9d56ec12253 2024-11-24T04:25:52,805 INFO [StoreOpener-478fe28262f058acb865d9d56ec12253-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 478fe28262f058acb865d9d56ec12253 columnFamilyName a 2024-11-24T04:25:52,805 DEBUG [StoreOpener-478fe28262f058acb865d9d56ec12253-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:25:52,805 INFO [StoreOpener-478fe28262f058acb865d9d56ec12253-1 {}] regionserver.HStore(327): Store=478fe28262f058acb865d9d56ec12253/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:25:52,806 INFO [StoreOpener-478fe28262f058acb865d9d56ec12253-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 478fe28262f058acb865d9d56ec12253 2024-11-24T04:25:52,808 INFO [StoreOpener-478fe28262f058acb865d9d56ec12253-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 478fe28262f058acb865d9d56ec12253 columnFamilyName b 2024-11-24T04:25:52,809 DEBUG [StoreOpener-478fe28262f058acb865d9d56ec12253-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:25:52,809 INFO [StoreOpener-478fe28262f058acb865d9d56ec12253-1 {}] regionserver.HStore(327): Store=478fe28262f058acb865d9d56ec12253/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:25:52,809 INFO [StoreOpener-478fe28262f058acb865d9d56ec12253-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 478fe28262f058acb865d9d56ec12253 2024-11-24T04:25:52,811 INFO [StoreOpener-478fe28262f058acb865d9d56ec12253-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 478fe28262f058acb865d9d56ec12253 columnFamilyName c 2024-11-24T04:25:52,811 DEBUG [StoreOpener-478fe28262f058acb865d9d56ec12253-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:25:52,811 INFO [StoreOpener-478fe28262f058acb865d9d56ec12253-1 {}] regionserver.HStore(327): Store=478fe28262f058acb865d9d56ec12253/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:25:52,812 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 478fe28262f058acb865d9d56ec12253 2024-11-24T04:25:52,812 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44611/hbase/data/default/testReplayEditsAfterAbortingFlush/478fe28262f058acb865d9d56ec12253 2024-11-24T04:25:52,813 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44611/hbase/data/default/testReplayEditsAfterAbortingFlush/478fe28262f058acb865d9d56ec12253 2024-11-24T04:25:52,814 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 478fe28262f058acb865d9d56ec12253 2024-11-24T04:25:52,814 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 478fe28262f058acb865d9d56ec12253 2024-11-24T04:25:52,814 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterAbortingFlush descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-24T04:25:52,815 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 478fe28262f058acb865d9d56ec12253 2024-11-24T04:25:52,818 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44611/hbase/data/default/testReplayEditsAfterAbortingFlush/478fe28262f058acb865d9d56ec12253/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T04:25:52,818 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 478fe28262f058acb865d9d56ec12253; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72293148, jitterRate=0.0772518515586853}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-24T04:25:52,819 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 478fe28262f058acb865d9d56ec12253: Writing region info on filesystem at 1732422352802Initializing all the Stores at 1732422352803 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732422352803Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732422352803Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732422352803Cleaning up temporary data from old regions at 1732422352814 (+11 ms)Region opened successfully at 1732422352819 (+5 ms) 2024-11-24T04:25:52,819 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 478fe28262f058acb865d9d56ec12253, disabling compactions & flushes 2024-11-24T04:25:52,819 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterAbortingFlush,,1732422352758.478fe28262f058acb865d9d56ec12253. 2024-11-24T04:25:52,820 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterAbortingFlush,,1732422352758.478fe28262f058acb865d9d56ec12253. 2024-11-24T04:25:52,820 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterAbortingFlush,,1732422352758.478fe28262f058acb865d9d56ec12253. after waiting 0 ms 2024-11-24T04:25:52,820 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterAbortingFlush,,1732422352758.478fe28262f058acb865d9d56ec12253. 2024-11-24T04:25:52,820 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsAfterAbortingFlush,,1732422352758.478fe28262f058acb865d9d56ec12253. 2024-11-24T04:25:52,820 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 478fe28262f058acb865d9d56ec12253: Waiting for close lock at 1732422352819Disabling compacts and flushes for region at 1732422352819Disabling writes for close at 1732422352820 (+1 ms)Writing region close event to WAL at 1732422352820Closed at 1732422352820 2024-11-24T04:25:52,824 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34457 is added to blk_1073741901_1079 (size=95) 2024-11-24T04:25:52,824 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36069 is added to blk_1073741901_1079 (size=95) 2024-11-24T04:25:52,824 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44781 is added to blk_1073741901_1079 (size=95) 2024-11-24T04:25:52,827 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-24T04:25:52,827 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-07185885:(num 1732422352762) 2024-11-24T04:25:52,827 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-24T04:25:52,829 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:44611/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732422352756, archiveDir=hdfs://localhost:44611/hbase/oldWALs, maxLogs=32 2024-11-24T04:25:52,844 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732422352756/wal.1732422352830, exclude list is [], retry=0 2024-11-24T04:25:52,847 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36069,DS-11c8f594-4fc2-47e4-b222-092ec50eb991,DISK] 2024-11-24T04:25:52,848 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44781,DS-54254ac8-0fae-43f7-ab06-53faa0e05dd7,DISK] 2024-11-24T04:25:52,848 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34457,DS-20bb5930-23a3-441d-bdb2-015c0d56b2ec,DISK] 2024-11-24T04:25:52,851 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732422352756/wal.1732422352830 2024-11-24T04:25:52,852 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41009:41009),(127.0.0.1/127.0.0.1:35691:35691),(127.0.0.1/127.0.0.1:33347:33347)] 2024-11-24T04:25:52,921 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 478fe28262f058acb865d9d56ec12253, NAME => 'testReplayEditsAfterAbortingFlush,,1732422352758.478fe28262f058acb865d9d56ec12253.', STARTKEY => '', ENDKEY => ''} 2024-11-24T04:25:52,924 DEBUG [Time-limited test {}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterAbortingFlush 478fe28262f058acb865d9d56ec12253 2024-11-24T04:25:52,924 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterAbortingFlush,,1732422352758.478fe28262f058acb865d9d56ec12253.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T04:25:52,924 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 478fe28262f058acb865d9d56ec12253 2024-11-24T04:25:52,924 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 478fe28262f058acb865d9d56ec12253 2024-11-24T04:25:52,926 INFO [StoreOpener-478fe28262f058acb865d9d56ec12253-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 478fe28262f058acb865d9d56ec12253 2024-11-24T04:25:52,927 INFO [StoreOpener-478fe28262f058acb865d9d56ec12253-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 478fe28262f058acb865d9d56ec12253 columnFamilyName a 2024-11-24T04:25:52,927 DEBUG [StoreOpener-478fe28262f058acb865d9d56ec12253-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:25:52,928 INFO [StoreOpener-478fe28262f058acb865d9d56ec12253-1 {}] regionserver.HStore(327): Store=478fe28262f058acb865d9d56ec12253/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:25:52,928 INFO [StoreOpener-478fe28262f058acb865d9d56ec12253-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 478fe28262f058acb865d9d56ec12253 2024-11-24T04:25:52,929 INFO [StoreOpener-478fe28262f058acb865d9d56ec12253-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 478fe28262f058acb865d9d56ec12253 columnFamilyName b 2024-11-24T04:25:52,929 DEBUG [StoreOpener-478fe28262f058acb865d9d56ec12253-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:25:52,930 INFO [StoreOpener-478fe28262f058acb865d9d56ec12253-1 {}] regionserver.HStore(327): Store=478fe28262f058acb865d9d56ec12253/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:25:52,930 INFO [StoreOpener-478fe28262f058acb865d9d56ec12253-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 478fe28262f058acb865d9d56ec12253 2024-11-24T04:25:52,931 INFO [StoreOpener-478fe28262f058acb865d9d56ec12253-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 478fe28262f058acb865d9d56ec12253 columnFamilyName c 2024-11-24T04:25:52,931 DEBUG [StoreOpener-478fe28262f058acb865d9d56ec12253-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:25:52,931 INFO [StoreOpener-478fe28262f058acb865d9d56ec12253-1 {}] regionserver.HStore(327): Store=478fe28262f058acb865d9d56ec12253/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:25:52,932 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 478fe28262f058acb865d9d56ec12253 2024-11-24T04:25:52,932 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44611/hbase/data/default/testReplayEditsAfterAbortingFlush/478fe28262f058acb865d9d56ec12253 2024-11-24T04:25:52,933 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44611/hbase/data/default/testReplayEditsAfterAbortingFlush/478fe28262f058acb865d9d56ec12253 2024-11-24T04:25:52,934 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 478fe28262f058acb865d9d56ec12253 2024-11-24T04:25:52,934 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 478fe28262f058acb865d9d56ec12253 2024-11-24T04:25:52,935 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterAbortingFlush descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-24T04:25:52,936 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 478fe28262f058acb865d9d56ec12253 2024-11-24T04:25:52,937 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 478fe28262f058acb865d9d56ec12253; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59317712, jitterRate=-0.11609721183776855}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-24T04:25:52,937 DEBUG [Time-limited test {}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 478fe28262f058acb865d9d56ec12253 2024-11-24T04:25:52,938 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 478fe28262f058acb865d9d56ec12253: Running coprocessor pre-open hook at 1732422352924Writing region info on filesystem at 1732422352924Initializing all the Stores at 1732422352925 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732422352925Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732422352926 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732422352926Cleaning up temporary data from old regions at 1732422352934 (+8 ms)Running coprocessor post-open hooks at 1732422352937 (+3 ms)Region opened successfully at 1732422352938 (+1 ms) 2024-11-24T04:25:52,951 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 478fe28262f058acb865d9d56ec12253 3/3 column families, dataSize=590 B heapSize=2.08 KB 2024-11-24T04:25:52,952 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 478fe28262f058acb865d9d56ec12253/a, retrying num=0 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T04:25:53,309 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenViaHRegion 2024-11-24T04:25:53,309 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenViaHRegion Metrics about Tables on a single HBase RegionServer 2024-11-24T04:25:53,311 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testSequentialEditLogSeqNum 2024-11-24T04:25:53,311 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testSequentialEditLogSeqNum Metrics about Tables on a single HBase RegionServer 2024-11-24T04:25:53,312 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterRegionMovedWithMultiCF 2024-11-24T04:25:53,312 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterRegionMovedWithMultiCF Metrics about Tables on a single HBase RegionServer 2024-11-24T04:25:53,313 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRegionMadeOfBulkLoadedFilesOnly 2024-11-24T04:25:53,313 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testRegionMadeOfBulkLoadedFilesOnly Metrics about Tables on a single HBase RegionServer 2024-11-24T04:25:53,315 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterAbortingFlush 2024-11-24T04:25:53,315 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterAbortingFlush Metrics about Tables on a single HBase RegionServer 2024-11-24T04:25:53,953 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 478fe28262f058acb865d9d56ec12253/a, retrying num=1 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T04:25:54,718 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-24T04:25:54,954 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 478fe28262f058acb865d9d56ec12253/a, retrying num=2 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T04:25:55,955 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 478fe28262f058acb865d9d56ec12253/a, retrying num=3 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T04:25:56,955 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 478fe28262f058acb865d9d56ec12253/a, retrying num=4 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T04:25:57,956 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 478fe28262f058acb865d9d56ec12253/a, retrying num=5 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T04:25:58,957 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 478fe28262f058acb865d9d56ec12253/a, retrying num=6 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T04:25:59,957 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 478fe28262f058acb865d9d56ec12253/a, retrying num=7 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T04:26:00,958 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 478fe28262f058acb865d9d56ec12253/a, retrying num=8 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T04:26:01,959 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 478fe28262f058acb865d9d56ec12253/a, retrying num=9 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T04:26:01,960 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 478fe28262f058acb865d9d56ec12253: 2024-11-24T04:26:01,960 INFO [Time-limited test {}] wal.AbstractTestWALReplay(671): Expected simulated exception when flushing region, region: testReplayEditsAfterAbortingFlush,,1732422352758.478fe28262f058acb865d9d56ec12253. 2024-11-24T04:26:01,976 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 478fe28262f058acb865d9d56ec12253: 2024-11-24T04:26:01,976 INFO [Time-limited test {}] wal.AbstractTestWALReplay(691): Expected exception when flushing region because server is stopped,Aborting flush because server is aborted... 2024-11-24T04:26:01,976 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 478fe28262f058acb865d9d56ec12253, disabling compactions & flushes 2024-11-24T04:26:01,976 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterAbortingFlush,,1732422352758.478fe28262f058acb865d9d56ec12253. 2024-11-24T04:26:01,976 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterAbortingFlush,,1732422352758.478fe28262f058acb865d9d56ec12253. 2024-11-24T04:26:01,976 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterAbortingFlush,,1732422352758.478fe28262f058acb865d9d56ec12253. after waiting 0 ms 2024-11-24T04:26:01,976 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterAbortingFlush,,1732422352758.478fe28262f058acb865d9d56ec12253. 2024-11-24T04:26:01,977 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 1190 in region testReplayEditsAfterAbortingFlush,,1732422352758.478fe28262f058acb865d9d56ec12253. 2024-11-24T04:26:01,977 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsAfterAbortingFlush,,1732422352758.478fe28262f058acb865d9d56ec12253. 2024-11-24T04:26:01,977 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 478fe28262f058acb865d9d56ec12253: Waiting for close lock at 1732422361976Running coprocessor pre-close hooks at 1732422361976Disabling compacts and flushes for region at 1732422361976Disabling writes for close at 1732422361976Writing region close event to WAL at 1732422361976Running coprocessor post-close hooks at 1732422361977 (+1 ms)Closed at 1732422361977 2024-11-24T04:26:01,979 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34457 is added to blk_1073741903_1081 (size=2691) 2024-11-24T04:26:01,979 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36069 is added to blk_1073741903_1081 (size=2691) 2024-11-24T04:26:01,980 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44781 is added to blk_1073741903_1081 (size=2691) 2024-11-24T04:26:02,004 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:44611/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732422352756/wal.1732422352830, size=2.6 K (2691bytes) 2024-11-24T04:26:02,004 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:44611/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732422352756/wal.1732422352830 2024-11-24T04:26:02,004 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:44611/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732422352756/wal.1732422352830 after 0ms 2024-11-24T04:26:02,007 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:44611/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732422352756/wal.1732422352830: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-24T04:26:02,008 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:44611/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732422352756/wal.1732422352830 took 4ms 2024-11-24T04:26:02,010 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:44611/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732422352756/wal.1732422352830 so closing down 2024-11-24T04:26:02,010 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-24T04:26:02,011 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-wal.1732422352830.temp 2024-11-24T04:26:02,013 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:44611/hbase/data/default/testReplayEditsAfterAbortingFlush/478fe28262f058acb865d9d56ec12253/recovered.edits/0000000000000000004-wal.1732422352830.temp 2024-11-24T04:26:02,013 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-24T04:26:02,020 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36069 is added to blk_1073741904_1082 (size=2094) 2024-11-24T04:26:02,020 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34457 is added to blk_1073741904_1082 (size=2094) 2024-11-24T04:26:02,021 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44781 is added to blk_1073741904_1082 (size=2094) 2024-11-24T04:26:02,024 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-24T04:26:02,025 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:44611/hbase/data/default/testReplayEditsAfterAbortingFlush/478fe28262f058acb865d9d56ec12253/recovered.edits/0000000000000000004-wal.1732422352830.temp (wrote 20 edits, skipped 0 edits in 0 ms) 2024-11-24T04:26:02,026 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:44611/hbase/data/default/testReplayEditsAfterAbortingFlush/478fe28262f058acb865d9d56ec12253/recovered.edits/0000000000000000004-wal.1732422352830.temp to hdfs://localhost:44611/hbase/data/default/testReplayEditsAfterAbortingFlush/478fe28262f058acb865d9d56ec12253/recovered.edits/0000000000000000026 2024-11-24T04:26:02,026 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 23 edits across 1 Regions in 18 ms; skipped=3; WAL=hdfs://localhost:44611/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732422352756/wal.1732422352830, size=2.6 K, length=2691, corrupted=false, cancelled=false 2024-11-24T04:26:02,026 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:44611/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732422352756/wal.1732422352830, journal: Splitting hdfs://localhost:44611/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732422352756/wal.1732422352830, size=2.6 K (2691bytes) at 1732422362004Finishing writing output for hdfs://localhost:44611/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732422352756/wal.1732422352830 so closing down at 1732422362010 (+6 ms)Creating recovered edits writer path=hdfs://localhost:44611/hbase/data/default/testReplayEditsAfterAbortingFlush/478fe28262f058acb865d9d56ec12253/recovered.edits/0000000000000000004-wal.1732422352830.temp at 1732422362013 (+3 ms)3 split writer threads finished at 1732422362013Closed recovered edits writer path=hdfs://localhost:44611/hbase/data/default/testReplayEditsAfterAbortingFlush/478fe28262f058acb865d9d56ec12253/recovered.edits/0000000000000000004-wal.1732422352830.temp (wrote 20 edits, skipped 0 edits in 0 ms) at 1732422362025 (+12 ms)Rename recovered edits hdfs://localhost:44611/hbase/data/default/testReplayEditsAfterAbortingFlush/478fe28262f058acb865d9d56ec12253/recovered.edits/0000000000000000004-wal.1732422352830.temp to hdfs://localhost:44611/hbase/data/default/testReplayEditsAfterAbortingFlush/478fe28262f058acb865d9d56ec12253/recovered.edits/0000000000000000026 at 1732422362026 (+1 ms)Processed 23 edits across 1 Regions in 18 ms; skipped=3; WAL=hdfs://localhost:44611/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732422352756/wal.1732422352830, size=2.6 K, length=2691, corrupted=false, cancelled=false at 1732422362026 2024-11-24T04:26:02,028 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:44611/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732422352756/wal.1732422352830 to hdfs://localhost:44611/hbase/oldWALs/wal.1732422352830 2024-11-24T04:26:02,029 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:44611/hbase/data/default/testReplayEditsAfterAbortingFlush/478fe28262f058acb865d9d56ec12253/recovered.edits/0000000000000000026 2024-11-24T04:26:02,029 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-24T04:26:02,031 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:44611/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732422352756, archiveDir=hdfs://localhost:44611/hbase/oldWALs, maxLogs=32 2024-11-24T04:26:02,045 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732422352756/wal.1732422362031, exclude list is [], retry=0 2024-11-24T04:26:02,048 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36069,DS-11c8f594-4fc2-47e4-b222-092ec50eb991,DISK] 2024-11-24T04:26:02,048 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34457,DS-20bb5930-23a3-441d-bdb2-015c0d56b2ec,DISK] 2024-11-24T04:26:02,048 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44781,DS-54254ac8-0fae-43f7-ab06-53faa0e05dd7,DISK] 2024-11-24T04:26:02,050 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732422352756/wal.1732422362031 2024-11-24T04:26:02,050 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41009:41009),(127.0.0.1/127.0.0.1:33347:33347),(127.0.0.1/127.0.0.1:35691:35691)] 2024-11-24T04:26:02,050 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 478fe28262f058acb865d9d56ec12253, NAME => 'testReplayEditsAfterAbortingFlush,,1732422352758.478fe28262f058acb865d9d56ec12253.', STARTKEY => '', ENDKEY => ''} 2024-11-24T04:26:02,051 DEBUG [Time-limited test {}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterAbortingFlush 478fe28262f058acb865d9d56ec12253 2024-11-24T04:26:02,051 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterAbortingFlush,,1732422352758.478fe28262f058acb865d9d56ec12253.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T04:26:02,051 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 478fe28262f058acb865d9d56ec12253 2024-11-24T04:26:02,051 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 478fe28262f058acb865d9d56ec12253 2024-11-24T04:26:02,053 INFO [StoreOpener-478fe28262f058acb865d9d56ec12253-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 478fe28262f058acb865d9d56ec12253 2024-11-24T04:26:02,054 INFO [StoreOpener-478fe28262f058acb865d9d56ec12253-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 478fe28262f058acb865d9d56ec12253 columnFamilyName a 2024-11-24T04:26:02,054 DEBUG [StoreOpener-478fe28262f058acb865d9d56ec12253-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:26:02,055 INFO [StoreOpener-478fe28262f058acb865d9d56ec12253-1 {}] regionserver.HStore(327): Store=478fe28262f058acb865d9d56ec12253/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:26:02,055 INFO [StoreOpener-478fe28262f058acb865d9d56ec12253-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 478fe28262f058acb865d9d56ec12253 2024-11-24T04:26:02,056 INFO [StoreOpener-478fe28262f058acb865d9d56ec12253-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 478fe28262f058acb865d9d56ec12253 columnFamilyName b 2024-11-24T04:26:02,056 DEBUG [StoreOpener-478fe28262f058acb865d9d56ec12253-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:26:02,056 INFO [StoreOpener-478fe28262f058acb865d9d56ec12253-1 {}] regionserver.HStore(327): Store=478fe28262f058acb865d9d56ec12253/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:26:02,056 INFO [StoreOpener-478fe28262f058acb865d9d56ec12253-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 478fe28262f058acb865d9d56ec12253 2024-11-24T04:26:02,057 INFO [StoreOpener-478fe28262f058acb865d9d56ec12253-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 478fe28262f058acb865d9d56ec12253 columnFamilyName c 2024-11-24T04:26:02,057 DEBUG [StoreOpener-478fe28262f058acb865d9d56ec12253-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:26:02,058 INFO [StoreOpener-478fe28262f058acb865d9d56ec12253-1 {}] regionserver.HStore(327): Store=478fe28262f058acb865d9d56ec12253/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:26:02,058 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 478fe28262f058acb865d9d56ec12253 2024-11-24T04:26:02,058 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44611/hbase/data/default/testReplayEditsAfterAbortingFlush/478fe28262f058acb865d9d56ec12253 2024-11-24T04:26:02,060 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:44611/hbase/data/default/testReplayEditsAfterAbortingFlush/478fe28262f058acb865d9d56ec12253 2024-11-24T04:26:02,060 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:44611/hbase/data/default/testReplayEditsAfterAbortingFlush/478fe28262f058acb865d9d56ec12253/recovered.edits/0000000000000000026 2024-11-24T04:26:02,062 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:44611/hbase/data/default/testReplayEditsAfterAbortingFlush/478fe28262f058acb865d9d56ec12253/recovered.edits/0000000000000000026: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-24T04:26:02,064 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 20, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=26, path=hdfs://localhost:44611/hbase/data/default/testReplayEditsAfterAbortingFlush/478fe28262f058acb865d9d56ec12253/recovered.edits/0000000000000000026 2024-11-24T04:26:02,064 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 478fe28262f058acb865d9d56ec12253 3/3 column families, dataSize=1.16 KB heapSize=3.41 KB 2024-11-24T04:26:02,080 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44611/hbase/data/default/testReplayEditsAfterAbortingFlush/478fe28262f058acb865d9d56ec12253/.tmp/a/59553fe21cf14511ad50f7862557f3b9 is 64, key is testReplayEditsAfterAbortingFlush12/a:q/1732422361966/Put/seqid=0 2024-11-24T04:26:02,086 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34457 is added to blk_1073741906_1084 (size=5523) 2024-11-24T04:26:02,087 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36069 is added to blk_1073741906_1084 (size=5523) 2024-11-24T04:26:02,087 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44781 is added to blk_1073741906_1084 (size=5523) 2024-11-24T04:26:02,088 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=416 B at sequenceid=26 (bloomFilter=true), to=hdfs://localhost:44611/hbase/data/default/testReplayEditsAfterAbortingFlush/478fe28262f058acb865d9d56ec12253/.tmp/a/59553fe21cf14511ad50f7862557f3b9 2024-11-24T04:26:02,108 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44611/hbase/data/default/testReplayEditsAfterAbortingFlush/478fe28262f058acb865d9d56ec12253/.tmp/b/a31203aa8fe042998b33e42632e1bec2 is 64, key is testReplayEditsAfterAbortingFlush10/b:q/1732422361961/Put/seqid=0 2024-11-24T04:26:02,115 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34457 is added to blk_1073741907_1085 (size=5524) 2024-11-24T04:26:02,115 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44781 is added to blk_1073741907_1085 (size=5524) 2024-11-24T04:26:02,115 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36069 is added to blk_1073741907_1085 (size=5524) 2024-11-24T04:26:02,116 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=417 B at sequenceid=26 (bloomFilter=true), to=hdfs://localhost:44611/hbase/data/default/testReplayEditsAfterAbortingFlush/478fe28262f058acb865d9d56ec12253/.tmp/b/a31203aa8fe042998b33e42632e1bec2 2024-11-24T04:26:02,138 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44611/hbase/data/default/testReplayEditsAfterAbortingFlush/478fe28262f058acb865d9d56ec12253/.tmp/c/adc6937decb24dbea5d8bdfc2f966230 is 64, key is testReplayEditsAfterAbortingFlush11/c:q/1732422361964/Put/seqid=0 2024-11-24T04:26:02,145 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34457 is added to blk_1073741908_1086 (size=5457) 2024-11-24T04:26:02,145 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36069 is added to blk_1073741908_1086 (size=5457) 2024-11-24T04:26:02,146 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44781 is added to blk_1073741908_1086 (size=5457) 2024-11-24T04:26:02,146 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=357 B at sequenceid=26 (bloomFilter=true), to=hdfs://localhost:44611/hbase/data/default/testReplayEditsAfterAbortingFlush/478fe28262f058acb865d9d56ec12253/.tmp/c/adc6937decb24dbea5d8bdfc2f966230 2024-11-24T04:26:02,153 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44611/hbase/data/default/testReplayEditsAfterAbortingFlush/478fe28262f058acb865d9d56ec12253/.tmp/a/59553fe21cf14511ad50f7862557f3b9 as hdfs://localhost:44611/hbase/data/default/testReplayEditsAfterAbortingFlush/478fe28262f058acb865d9d56ec12253/a/59553fe21cf14511ad50f7862557f3b9 2024-11-24T04:26:02,159 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44611/hbase/data/default/testReplayEditsAfterAbortingFlush/478fe28262f058acb865d9d56ec12253/a/59553fe21cf14511ad50f7862557f3b9, entries=7, sequenceid=26, filesize=5.4 K 2024-11-24T04:26:02,161 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44611/hbase/data/default/testReplayEditsAfterAbortingFlush/478fe28262f058acb865d9d56ec12253/.tmp/b/a31203aa8fe042998b33e42632e1bec2 as hdfs://localhost:44611/hbase/data/default/testReplayEditsAfterAbortingFlush/478fe28262f058acb865d9d56ec12253/b/a31203aa8fe042998b33e42632e1bec2 2024-11-24T04:26:02,166 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44611/hbase/data/default/testReplayEditsAfterAbortingFlush/478fe28262f058acb865d9d56ec12253/b/a31203aa8fe042998b33e42632e1bec2, entries=7, sequenceid=26, filesize=5.4 K 2024-11-24T04:26:02,167 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44611/hbase/data/default/testReplayEditsAfterAbortingFlush/478fe28262f058acb865d9d56ec12253/.tmp/c/adc6937decb24dbea5d8bdfc2f966230 as hdfs://localhost:44611/hbase/data/default/testReplayEditsAfterAbortingFlush/478fe28262f058acb865d9d56ec12253/c/adc6937decb24dbea5d8bdfc2f966230 2024-11-24T04:26:02,173 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44611/hbase/data/default/testReplayEditsAfterAbortingFlush/478fe28262f058acb865d9d56ec12253/c/adc6937decb24dbea5d8bdfc2f966230, entries=6, sequenceid=26, filesize=5.3 K 2024-11-24T04:26:02,173 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~1.16 KB/1190, heapSize ~3.36 KB/3440, currentSize=0 B/0 for 478fe28262f058acb865d9d56ec12253 in 109ms, sequenceid=26, compaction requested=false; wal=null 2024-11-24T04:26:02,174 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:44611/hbase/data/default/testReplayEditsAfterAbortingFlush/478fe28262f058acb865d9d56ec12253/recovered.edits/0000000000000000026 2024-11-24T04:26:02,175 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 478fe28262f058acb865d9d56ec12253 2024-11-24T04:26:02,175 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 478fe28262f058acb865d9d56ec12253 2024-11-24T04:26:02,176 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterAbortingFlush descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-24T04:26:02,178 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 478fe28262f058acb865d9d56ec12253 2024-11-24T04:26:02,180 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44611/hbase/data/default/testReplayEditsAfterAbortingFlush/478fe28262f058acb865d9d56ec12253/recovered.edits/26.seqid, newMaxSeqId=26, maxSeqId=1 2024-11-24T04:26:02,181 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 478fe28262f058acb865d9d56ec12253; next sequenceid=27; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68545729, jitterRate=0.021410956978797913}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-24T04:26:02,181 DEBUG [Time-limited test {}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 478fe28262f058acb865d9d56ec12253 2024-11-24T04:26:02,182 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 478fe28262f058acb865d9d56ec12253: Running coprocessor pre-open hook at 1732422362051Writing region info on filesystem at 1732422362051Initializing all the Stores at 1732422362052 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732422362052Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732422362053 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732422362053Obtaining lock to block concurrent updates at 1732422362064 (+11 ms)Preparing flush snapshotting stores in 478fe28262f058acb865d9d56ec12253 at 1732422362064Finished memstore snapshotting testReplayEditsAfterAbortingFlush,,1732422352758.478fe28262f058acb865d9d56ec12253., syncing WAL and waiting on mvcc, flushsize=dataSize=1190, getHeapSize=3440, getOffHeapSize=0, getCellsCount=20 at 1732422362064Flushing stores of testReplayEditsAfterAbortingFlush,,1732422352758.478fe28262f058acb865d9d56ec12253. at 1732422362064Flushing 478fe28262f058acb865d9d56ec12253/a: creating writer at 1732422362064Flushing 478fe28262f058acb865d9d56ec12253/a: appending metadata at 1732422362080 (+16 ms)Flushing 478fe28262f058acb865d9d56ec12253/a: closing flushed file at 1732422362080Flushing 478fe28262f058acb865d9d56ec12253/b: creating writer at 1732422362093 (+13 ms)Flushing 478fe28262f058acb865d9d56ec12253/b: appending metadata at 1732422362108 (+15 ms)Flushing 478fe28262f058acb865d9d56ec12253/b: closing flushed file at 1732422362108Flushing 478fe28262f058acb865d9d56ec12253/c: creating writer at 1732422362121 (+13 ms)Flushing 478fe28262f058acb865d9d56ec12253/c: appending metadata at 1732422362138 (+17 ms)Flushing 478fe28262f058acb865d9d56ec12253/c: closing flushed file at 1732422362138Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@271060b7: reopening flushed file at 1732422362152 (+14 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@226f3265: reopening flushed file at 1732422362160 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@70234b76: reopening flushed file at 1732422362166 (+6 ms)Finished flush of dataSize ~1.16 KB/1190, heapSize ~3.36 KB/3440, currentSize=0 B/0 for 478fe28262f058acb865d9d56ec12253 in 109ms, sequenceid=26, compaction requested=false; wal=null at 1732422362174 (+8 ms)Cleaning up temporary data from old regions at 1732422362175 (+1 ms)Running coprocessor post-open hooks at 1732422362181 (+6 ms)Region opened successfully at 1732422362182 (+1 ms) 2024-11-24T04:26:02,205 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayValueCompression#testReplayEditsAfterAbortingFlush Thread=420 (was 420), OpenFileDescriptor=1182 (was 1124) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=166 (was 196), ProcessCount=11 (was 11), AvailableMemoryMB=6133 (was 6149) 2024-11-24T04:26:02,205 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1182 is superior to 1024 2024-11-24T04:26:02,220 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayValueCompression#testDatalossWhenInputError Thread=420, OpenFileDescriptor=1182, MaxFileDescriptor=1048576, SystemLoadAverage=166, ProcessCount=11, AvailableMemoryMB=6131 2024-11-24T04:26:02,220 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1182 is superior to 1024 2024-11-24T04:26:02,237 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T04:26:02,239 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T04:26:02,239 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-24T04:26:02,242 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-12751718, suffix=, logDir=hdfs://localhost:44611/hbase/WALs/hregion-12751718, archiveDir=hdfs://localhost:44611/hbase/oldWALs, maxLogs=32 2024-11-24T04:26:02,255 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-12751718/hregion-12751718.1732422362242, exclude list is [], retry=0 2024-11-24T04:26:02,258 DEBUG [AsyncFSWAL-24-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34457,DS-20bb5930-23a3-441d-bdb2-015c0d56b2ec,DISK] 2024-11-24T04:26:02,258 DEBUG [AsyncFSWAL-24-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44781,DS-54254ac8-0fae-43f7-ab06-53faa0e05dd7,DISK] 2024-11-24T04:26:02,258 DEBUG [AsyncFSWAL-24-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36069,DS-11c8f594-4fc2-47e4-b222-092ec50eb991,DISK] 2024-11-24T04:26:02,261 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-12751718/hregion-12751718.1732422362242 2024-11-24T04:26:02,261 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33347:33347),(127.0.0.1/127.0.0.1:35691:35691),(127.0.0.1/127.0.0.1:41009:41009)] 2024-11-24T04:26:02,261 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 43fa644bf3cfae2d63593c3e88d815b0, NAME => 'testDatalossWhenInputError,,1732422362237.43fa644bf3cfae2d63593c3e88d815b0.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testDatalossWhenInputError', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44611/hbase 2024-11-24T04:26:02,268 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34457 is added to blk_1073741910_1088 (size=61) 2024-11-24T04:26:02,268 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44781 is added to blk_1073741910_1088 (size=61) 2024-11-24T04:26:02,269 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36069 is added to blk_1073741910_1088 (size=61) 2024-11-24T04:26:02,270 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1732422362237.43fa644bf3cfae2d63593c3e88d815b0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T04:26:02,271 INFO [StoreOpener-43fa644bf3cfae2d63593c3e88d815b0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 43fa644bf3cfae2d63593c3e88d815b0 2024-11-24T04:26:02,272 INFO [StoreOpener-43fa644bf3cfae2d63593c3e88d815b0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 43fa644bf3cfae2d63593c3e88d815b0 columnFamilyName a 2024-11-24T04:26:02,272 DEBUG [StoreOpener-43fa644bf3cfae2d63593c3e88d815b0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:26:02,273 INFO [StoreOpener-43fa644bf3cfae2d63593c3e88d815b0-1 {}] regionserver.HStore(327): Store=43fa644bf3cfae2d63593c3e88d815b0/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:26:02,273 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 43fa644bf3cfae2d63593c3e88d815b0 2024-11-24T04:26:02,274 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44611/hbase/data/default/testDatalossWhenInputError/43fa644bf3cfae2d63593c3e88d815b0 2024-11-24T04:26:02,274 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44611/hbase/data/default/testDatalossWhenInputError/43fa644bf3cfae2d63593c3e88d815b0 2024-11-24T04:26:02,275 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 43fa644bf3cfae2d63593c3e88d815b0 2024-11-24T04:26:02,275 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 43fa644bf3cfae2d63593c3e88d815b0 2024-11-24T04:26:02,277 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 43fa644bf3cfae2d63593c3e88d815b0 2024-11-24T04:26:02,279 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44611/hbase/data/default/testDatalossWhenInputError/43fa644bf3cfae2d63593c3e88d815b0/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T04:26:02,279 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 43fa644bf3cfae2d63593c3e88d815b0; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65023082, jitterRate=-0.031080573797225952}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-24T04:26:02,280 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 43fa644bf3cfae2d63593c3e88d815b0: Writing region info on filesystem at 1732422362270Initializing all the Stores at 1732422362271 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732422362271Cleaning up temporary data from old regions at 1732422362275 (+4 ms)Region opened successfully at 1732422362280 (+5 ms) 2024-11-24T04:26:02,280 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 43fa644bf3cfae2d63593c3e88d815b0, disabling compactions & flushes 2024-11-24T04:26:02,281 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testDatalossWhenInputError,,1732422362237.43fa644bf3cfae2d63593c3e88d815b0. 2024-11-24T04:26:02,281 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testDatalossWhenInputError,,1732422362237.43fa644bf3cfae2d63593c3e88d815b0. 2024-11-24T04:26:02,281 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testDatalossWhenInputError,,1732422362237.43fa644bf3cfae2d63593c3e88d815b0. after waiting 0 ms 2024-11-24T04:26:02,281 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testDatalossWhenInputError,,1732422362237.43fa644bf3cfae2d63593c3e88d815b0. 2024-11-24T04:26:02,281 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testDatalossWhenInputError,,1732422362237.43fa644bf3cfae2d63593c3e88d815b0. 2024-11-24T04:26:02,281 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 43fa644bf3cfae2d63593c3e88d815b0: Waiting for close lock at 1732422362280Disabling compacts and flushes for region at 1732422362280Disabling writes for close at 1732422362281 (+1 ms)Writing region close event to WAL at 1732422362281Closed at 1732422362281 2024-11-24T04:26:02,283 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44781 is added to blk_1073741909_1087 (size=95) 2024-11-24T04:26:02,284 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34457 is added to blk_1073741909_1087 (size=95) 2024-11-24T04:26:02,284 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36069 is added to blk_1073741909_1087 (size=95) 2024-11-24T04:26:02,286 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-24T04:26:02,286 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-12751718:(num 1732422362242) 2024-11-24T04:26:02,286 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-24T04:26:02,288 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:44611/hbase/WALs/testdatalosswheninputerror-manual,16010,1732422362236, archiveDir=hdfs://localhost:44611/hbase/oldWALs, maxLogs=32 2024-11-24T04:26:02,302 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testdatalosswheninputerror-manual,16010,1732422362236/wal.1732422362289, exclude list is [], retry=0 2024-11-24T04:26:02,305 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34457,DS-20bb5930-23a3-441d-bdb2-015c0d56b2ec,DISK] 2024-11-24T04:26:02,305 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36069,DS-11c8f594-4fc2-47e4-b222-092ec50eb991,DISK] 2024-11-24T04:26:02,305 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44781,DS-54254ac8-0fae-43f7-ab06-53faa0e05dd7,DISK] 2024-11-24T04:26:02,307 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testdatalosswheninputerror-manual,16010,1732422362236/wal.1732422362289 2024-11-24T04:26:02,307 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33347:33347),(127.0.0.1/127.0.0.1:41009:41009),(127.0.0.1/127.0.0.1:35691:35691)] 2024-11-24T04:26:02,307 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 43fa644bf3cfae2d63593c3e88d815b0, NAME => 'testDatalossWhenInputError,,1732422362237.43fa644bf3cfae2d63593c3e88d815b0.', STARTKEY => '', ENDKEY => ''} 2024-11-24T04:26:02,307 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1732422362237.43fa644bf3cfae2d63593c3e88d815b0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T04:26:02,307 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 43fa644bf3cfae2d63593c3e88d815b0 2024-11-24T04:26:02,307 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 43fa644bf3cfae2d63593c3e88d815b0 2024-11-24T04:26:02,309 INFO [StoreOpener-43fa644bf3cfae2d63593c3e88d815b0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 43fa644bf3cfae2d63593c3e88d815b0 2024-11-24T04:26:02,310 INFO [StoreOpener-43fa644bf3cfae2d63593c3e88d815b0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 43fa644bf3cfae2d63593c3e88d815b0 columnFamilyName a 2024-11-24T04:26:02,310 DEBUG [StoreOpener-43fa644bf3cfae2d63593c3e88d815b0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:26:02,310 INFO [StoreOpener-43fa644bf3cfae2d63593c3e88d815b0-1 {}] regionserver.HStore(327): Store=43fa644bf3cfae2d63593c3e88d815b0/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:26:02,310 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 43fa644bf3cfae2d63593c3e88d815b0 2024-11-24T04:26:02,311 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44611/hbase/data/default/testDatalossWhenInputError/43fa644bf3cfae2d63593c3e88d815b0 2024-11-24T04:26:02,312 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44611/hbase/data/default/testDatalossWhenInputError/43fa644bf3cfae2d63593c3e88d815b0 2024-11-24T04:26:02,312 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 43fa644bf3cfae2d63593c3e88d815b0 2024-11-24T04:26:02,312 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 43fa644bf3cfae2d63593c3e88d815b0 2024-11-24T04:26:02,314 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 43fa644bf3cfae2d63593c3e88d815b0 2024-11-24T04:26:02,315 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 43fa644bf3cfae2d63593c3e88d815b0; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73222280, jitterRate=0.09109699726104736}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-24T04:26:02,316 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 43fa644bf3cfae2d63593c3e88d815b0: Writing region info on filesystem at 1732422362308Initializing all the Stores at 1732422362308Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732422362308Cleaning up temporary data from old regions at 1732422362312 (+4 ms)Region opened successfully at 1732422362315 (+3 ms) 2024-11-24T04:26:02,323 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 43fa644bf3cfae2d63593c3e88d815b0, disabling compactions & flushes 2024-11-24T04:26:02,323 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testDatalossWhenInputError,,1732422362237.43fa644bf3cfae2d63593c3e88d815b0. 2024-11-24T04:26:02,324 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testDatalossWhenInputError,,1732422362237.43fa644bf3cfae2d63593c3e88d815b0. 2024-11-24T04:26:02,324 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testDatalossWhenInputError,,1732422362237.43fa644bf3cfae2d63593c3e88d815b0. after waiting 0 ms 2024-11-24T04:26:02,324 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testDatalossWhenInputError,,1732422362237.43fa644bf3cfae2d63593c3e88d815b0. 2024-11-24T04:26:02,324 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 750 in region testDatalossWhenInputError,,1732422362237.43fa644bf3cfae2d63593c3e88d815b0. 2024-11-24T04:26:02,324 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testDatalossWhenInputError,,1732422362237.43fa644bf3cfae2d63593c3e88d815b0. 2024-11-24T04:26:02,324 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 43fa644bf3cfae2d63593c3e88d815b0: Waiting for close lock at 1732422362323Disabling compacts and flushes for region at 1732422362323Disabling writes for close at 1732422362324 (+1 ms)Writing region close event to WAL at 1732422362324Closed at 1732422362324 2024-11-24T04:26:02,330 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36069 is added to blk_1073741911_1089 (size=1050) 2024-11-24T04:26:02,330 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34457 is added to blk_1073741911_1089 (size=1050) 2024-11-24T04:26:02,331 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44781 is added to blk_1073741911_1089 (size=1050) 2024-11-24T04:26:02,347 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:44611/hbase/WALs/testdatalosswheninputerror-manual,16010,1732422362236/wal.1732422362289, size=1.0 K (1050bytes) 2024-11-24T04:26:02,347 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:44611/hbase/WALs/testdatalosswheninputerror-manual,16010,1732422362236/wal.1732422362289 2024-11-24T04:26:02,348 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:44611/hbase/WALs/testdatalosswheninputerror-manual,16010,1732422362236/wal.1732422362289 after 1ms 2024-11-24T04:26:02,350 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:44611/hbase/WALs/testdatalosswheninputerror-manual,16010,1732422362236/wal.1732422362289: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-24T04:26:02,350 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:44611/hbase/WALs/testdatalosswheninputerror-manual,16010,1732422362236/wal.1732422362289 took 3ms 2024-11-24T04:26:02,352 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:44611/hbase/WALs/testdatalosswheninputerror-manual,16010,1732422362236/wal.1732422362289 so closing down 2024-11-24T04:26:02,352 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-24T04:26:02,354 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1732422362289.temp 2024-11-24T04:26:02,355 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:44611/hbase/data/default/testDatalossWhenInputError/43fa644bf3cfae2d63593c3e88d815b0/recovered.edits/0000000000000000003-wal.1732422362289.temp 2024-11-24T04:26:02,355 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-24T04:26:02,361 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44781 is added to blk_1073741912_1090 (size=1050) 2024-11-24T04:26:02,361 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36069 is added to blk_1073741912_1090 (size=1050) 2024-11-24T04:26:02,362 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:44611/hbase/data/default/testDatalossWhenInputError/43fa644bf3cfae2d63593c3e88d815b0/recovered.edits/0000000000000000003-wal.1732422362289.temp (wrote 10 edits, skipped 0 edits in 0 ms) 2024-11-24T04:26:02,362 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34457 is added to blk_1073741912_1090 (size=1050) 2024-11-24T04:26:02,363 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:44611/hbase/data/default/testDatalossWhenInputError/43fa644bf3cfae2d63593c3e88d815b0/recovered.edits/0000000000000000003-wal.1732422362289.temp to hdfs://localhost:44611/hbase/data/default/testDatalossWhenInputError/43fa644bf3cfae2d63593c3e88d815b0/recovered.edits/0000000000000000012 2024-11-24T04:26:02,363 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 10 edits across 1 Regions in 12 ms; skipped=0; WAL=hdfs://localhost:44611/hbase/WALs/testdatalosswheninputerror-manual,16010,1732422362236/wal.1732422362289, size=1.0 K, length=1050, corrupted=false, cancelled=false 2024-11-24T04:26:02,363 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:44611/hbase/WALs/testdatalosswheninputerror-manual,16010,1732422362236/wal.1732422362289, journal: Splitting hdfs://localhost:44611/hbase/WALs/testdatalosswheninputerror-manual,16010,1732422362236/wal.1732422362289, size=1.0 K (1050bytes) at 1732422362347Finishing writing output for hdfs://localhost:44611/hbase/WALs/testdatalosswheninputerror-manual,16010,1732422362236/wal.1732422362289 so closing down at 1732422362352 (+5 ms)Creating recovered edits writer path=hdfs://localhost:44611/hbase/data/default/testDatalossWhenInputError/43fa644bf3cfae2d63593c3e88d815b0/recovered.edits/0000000000000000003-wal.1732422362289.temp at 1732422362355 (+3 ms)3 split writer threads finished at 1732422362355Closed recovered edits writer path=hdfs://localhost:44611/hbase/data/default/testDatalossWhenInputError/43fa644bf3cfae2d63593c3e88d815b0/recovered.edits/0000000000000000003-wal.1732422362289.temp (wrote 10 edits, skipped 0 edits in 0 ms) at 1732422362362 (+7 ms)Rename recovered edits hdfs://localhost:44611/hbase/data/default/testDatalossWhenInputError/43fa644bf3cfae2d63593c3e88d815b0/recovered.edits/0000000000000000003-wal.1732422362289.temp to hdfs://localhost:44611/hbase/data/default/testDatalossWhenInputError/43fa644bf3cfae2d63593c3e88d815b0/recovered.edits/0000000000000000012 at 1732422362363 (+1 ms)Processed 10 edits across 1 Regions in 12 ms; skipped=0; WAL=hdfs://localhost:44611/hbase/WALs/testdatalosswheninputerror-manual,16010,1732422362236/wal.1732422362289, size=1.0 K, length=1050, corrupted=false, cancelled=false at 1732422362363 2024-11-24T04:26:02,365 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:44611/hbase/WALs/testdatalosswheninputerror-manual,16010,1732422362236/wal.1732422362289 to hdfs://localhost:44611/hbase/oldWALs/wal.1732422362289 2024-11-24T04:26:02,366 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:44611/hbase/data/default/testDatalossWhenInputError/43fa644bf3cfae2d63593c3e88d815b0/recovered.edits/0000000000000000012 2024-11-24T04:26:02,369 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:44611/hbase/data/default/testDatalossWhenInputError/43fa644bf3cfae2d63593c3e88d815b0/recovered.edits/0000000000000000012: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-24T04:26:02,694 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-24T04:26:02,696 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:44611/hbase/WALs/testdatalosswheninputerror-manual,16010,1732422362236, archiveDir=hdfs://localhost:44611/hbase/oldWALs, maxLogs=32 2024-11-24T04:26:02,710 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testdatalosswheninputerror-manual,16010,1732422362236/wal.1732422362696, exclude list is [], retry=0 2024-11-24T04:26:02,713 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36069,DS-11c8f594-4fc2-47e4-b222-092ec50eb991,DISK] 2024-11-24T04:26:02,713 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34457,DS-20bb5930-23a3-441d-bdb2-015c0d56b2ec,DISK] 2024-11-24T04:26:02,714 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44781,DS-54254ac8-0fae-43f7-ab06-53faa0e05dd7,DISK] 2024-11-24T04:26:02,717 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testdatalosswheninputerror-manual,16010,1732422362236/wal.1732422362696 2024-11-24T04:26:02,717 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41009:41009),(127.0.0.1/127.0.0.1:33347:33347),(127.0.0.1/127.0.0.1:35691:35691)] 2024-11-24T04:26:02,717 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 43fa644bf3cfae2d63593c3e88d815b0, NAME => 'testDatalossWhenInputError,,1732422362237.43fa644bf3cfae2d63593c3e88d815b0.', STARTKEY => '', ENDKEY => ''} 2024-11-24T04:26:02,717 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1732422362237.43fa644bf3cfae2d63593c3e88d815b0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T04:26:02,718 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 43fa644bf3cfae2d63593c3e88d815b0 2024-11-24T04:26:02,718 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 43fa644bf3cfae2d63593c3e88d815b0 2024-11-24T04:26:02,722 INFO [StoreOpener-43fa644bf3cfae2d63593c3e88d815b0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 43fa644bf3cfae2d63593c3e88d815b0 2024-11-24T04:26:02,724 INFO [StoreOpener-43fa644bf3cfae2d63593c3e88d815b0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 43fa644bf3cfae2d63593c3e88d815b0 columnFamilyName a 2024-11-24T04:26:02,724 DEBUG [StoreOpener-43fa644bf3cfae2d63593c3e88d815b0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:26:02,725 INFO [StoreOpener-43fa644bf3cfae2d63593c3e88d815b0-1 {}] regionserver.HStore(327): Store=43fa644bf3cfae2d63593c3e88d815b0/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:26:02,725 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 43fa644bf3cfae2d63593c3e88d815b0 2024-11-24T04:26:02,726 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44611/hbase/data/default/testDatalossWhenInputError/43fa644bf3cfae2d63593c3e88d815b0 2024-11-24T04:26:02,728 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:44611/hbase/data/default/testDatalossWhenInputError/43fa644bf3cfae2d63593c3e88d815b0 2024-11-24T04:26:02,729 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:44611/hbase/data/default/testDatalossWhenInputError/43fa644bf3cfae2d63593c3e88d815b0/recovered.edits/0000000000000000012 2024-11-24T04:26:02,731 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:44611/hbase/data/default/testDatalossWhenInputError/43fa644bf3cfae2d63593c3e88d815b0/recovered.edits/0000000000000000012: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-24T04:26:02,732 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 10, skipped 0, firstSequenceIdInLog=3, maxSequenceIdInLog=12, path=hdfs://localhost:44611/hbase/data/default/testDatalossWhenInputError/43fa644bf3cfae2d63593c3e88d815b0/recovered.edits/0000000000000000012 2024-11-24T04:26:02,733 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 43fa644bf3cfae2d63593c3e88d815b0 1/1 column families, dataSize=750 B heapSize=1.73 KB 2024-11-24T04:26:02,758 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44611/hbase/data/default/testDatalossWhenInputError/43fa644bf3cfae2d63593c3e88d815b0/.tmp/a/5a46b2653b734c2d89d6ce2c267b06c9 is 79, key is testDatalossWhenInputError/a:x0/1732422362316/Put/seqid=0 2024-11-24T04:26:02,766 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34457 is added to blk_1073741914_1092 (size=5808) 2024-11-24T04:26:02,767 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36069 is added to blk_1073741914_1092 (size=5808) 2024-11-24T04:26:02,767 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44781 is added to blk_1073741914_1092 (size=5808) 2024-11-24T04:26:02,767 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=750 B at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:44611/hbase/data/default/testDatalossWhenInputError/43fa644bf3cfae2d63593c3e88d815b0/.tmp/a/5a46b2653b734c2d89d6ce2c267b06c9 2024-11-24T04:26:02,778 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44611/hbase/data/default/testDatalossWhenInputError/43fa644bf3cfae2d63593c3e88d815b0/.tmp/a/5a46b2653b734c2d89d6ce2c267b06c9 as hdfs://localhost:44611/hbase/data/default/testDatalossWhenInputError/43fa644bf3cfae2d63593c3e88d815b0/a/5a46b2653b734c2d89d6ce2c267b06c9 2024-11-24T04:26:02,788 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44611/hbase/data/default/testDatalossWhenInputError/43fa644bf3cfae2d63593c3e88d815b0/a/5a46b2653b734c2d89d6ce2c267b06c9, entries=10, sequenceid=12, filesize=5.7 K 2024-11-24T04:26:02,788 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~750 B/750, heapSize ~1.72 KB/1760, currentSize=0 B/0 for 43fa644bf3cfae2d63593c3e88d815b0 in 55ms, sequenceid=12, compaction requested=false; wal=null 2024-11-24T04:26:02,789 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:44611/hbase/data/default/testDatalossWhenInputError/43fa644bf3cfae2d63593c3e88d815b0/recovered.edits/0000000000000000012 2024-11-24T04:26:02,790 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 43fa644bf3cfae2d63593c3e88d815b0 2024-11-24T04:26:02,790 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 43fa644bf3cfae2d63593c3e88d815b0 2024-11-24T04:26:02,792 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 43fa644bf3cfae2d63593c3e88d815b0 2024-11-24T04:26:02,794 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44611/hbase/data/default/testDatalossWhenInputError/43fa644bf3cfae2d63593c3e88d815b0/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=1 2024-11-24T04:26:02,795 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 43fa644bf3cfae2d63593c3e88d815b0; next sequenceid=13; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68572819, jitterRate=0.021814629435539246}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-24T04:26:02,796 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 43fa644bf3cfae2d63593c3e88d815b0: Writing region info on filesystem at 1732422362718Initializing all the Stores at 1732422362721 (+3 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732422362721Obtaining lock to block concurrent updates at 1732422362733 (+12 ms)Preparing flush snapshotting stores in 43fa644bf3cfae2d63593c3e88d815b0 at 1732422362733Finished memstore snapshotting testDatalossWhenInputError,,1732422362237.43fa644bf3cfae2d63593c3e88d815b0., syncing WAL and waiting on mvcc, flushsize=dataSize=750, getHeapSize=1760, getOffHeapSize=0, getCellsCount=10 at 1732422362733Flushing stores of testDatalossWhenInputError,,1732422362237.43fa644bf3cfae2d63593c3e88d815b0. at 1732422362733Flushing 43fa644bf3cfae2d63593c3e88d815b0/a: creating writer at 1732422362733Flushing 43fa644bf3cfae2d63593c3e88d815b0/a: appending metadata at 1732422362757 (+24 ms)Flushing 43fa644bf3cfae2d63593c3e88d815b0/a: closing flushed file at 1732422362757Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3dbbbd73: reopening flushed file at 1732422362776 (+19 ms)Finished flush of dataSize ~750 B/750, heapSize ~1.72 KB/1760, currentSize=0 B/0 for 43fa644bf3cfae2d63593c3e88d815b0 in 55ms, sequenceid=12, compaction requested=false; wal=null at 1732422362788 (+12 ms)Cleaning up temporary data from old regions at 1732422362790 (+2 ms)Region opened successfully at 1732422362796 (+6 ms) 2024-11-24T04:26:02,799 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 43fa644bf3cfae2d63593c3e88d815b0, NAME => 'testDatalossWhenInputError,,1732422362237.43fa644bf3cfae2d63593c3e88d815b0.', STARTKEY => '', ENDKEY => ''} 2024-11-24T04:26:02,799 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1732422362237.43fa644bf3cfae2d63593c3e88d815b0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T04:26:02,799 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 43fa644bf3cfae2d63593c3e88d815b0 2024-11-24T04:26:02,799 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 43fa644bf3cfae2d63593c3e88d815b0 2024-11-24T04:26:02,800 INFO [StoreOpener-43fa644bf3cfae2d63593c3e88d815b0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 43fa644bf3cfae2d63593c3e88d815b0 2024-11-24T04:26:02,801 INFO [StoreOpener-43fa644bf3cfae2d63593c3e88d815b0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 43fa644bf3cfae2d63593c3e88d815b0 columnFamilyName a 2024-11-24T04:26:02,801 DEBUG [StoreOpener-43fa644bf3cfae2d63593c3e88d815b0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:26:02,806 DEBUG [StoreOpener-43fa644bf3cfae2d63593c3e88d815b0-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:44611/hbase/data/default/testDatalossWhenInputError/43fa644bf3cfae2d63593c3e88d815b0/a/5a46b2653b734c2d89d6ce2c267b06c9 2024-11-24T04:26:02,806 INFO [StoreOpener-43fa644bf3cfae2d63593c3e88d815b0-1 {}] regionserver.HStore(327): Store=43fa644bf3cfae2d63593c3e88d815b0/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:26:02,806 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 43fa644bf3cfae2d63593c3e88d815b0 2024-11-24T04:26:02,807 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44611/hbase/data/default/testDatalossWhenInputError/43fa644bf3cfae2d63593c3e88d815b0 2024-11-24T04:26:02,808 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44611/hbase/data/default/testDatalossWhenInputError/43fa644bf3cfae2d63593c3e88d815b0 2024-11-24T04:26:02,809 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 43fa644bf3cfae2d63593c3e88d815b0 2024-11-24T04:26:02,809 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 43fa644bf3cfae2d63593c3e88d815b0 2024-11-24T04:26:02,810 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 43fa644bf3cfae2d63593c3e88d815b0 2024-11-24T04:26:02,813 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44611/hbase/data/default/testDatalossWhenInputError/43fa644bf3cfae2d63593c3e88d815b0/recovered.edits/13.seqid, newMaxSeqId=13, maxSeqId=12 2024-11-24T04:26:02,814 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 43fa644bf3cfae2d63593c3e88d815b0; next sequenceid=14; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72766445, jitterRate=0.08430452644824982}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-24T04:26:02,814 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 43fa644bf3cfae2d63593c3e88d815b0: Writing region info on filesystem at 1732422362799Initializing all the Stores at 1732422362800 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732422362800Cleaning up temporary data from old regions at 1732422362809 (+9 ms)Region opened successfully at 1732422362814 (+5 ms) 2024-11-24T04:26:02,832 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayValueCompression#testDatalossWhenInputError Thread=430 (was 420) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1575425110_22 at /127.0.0.1:45956 [Receiving block BP-1091946017-172.17.0.2-1732422329789:blk_1073741913_1091] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-24-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-24-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1575425110_22 at /127.0.0.1:58472 [Waiting for operation #20] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1091946017-172.17.0.2-1732422329789:blk_1073741913_1091, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1575425110_22 at /127.0.0.1:55316 [Receiving block BP-1091946017-172.17.0.2-1732422329789:blk_1073741913_1091] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1575425110_22 at /127.0.0.1:45888 [Waiting for operation #9] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1575425110_22 at /127.0.0.1:58546 [Receiving block BP-1091946017-172.17.0.2-1732422329789:blk_1073741913_1091] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1091946017-172.17.0.2-1732422329789:blk_1073741913_1091, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1575425110_22 at /127.0.0.1:55228 [Waiting for operation #13] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-24-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1091946017-172.17.0.2-1732422329789:blk_1073741913_1091, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1264 (was 1182) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=166 (was 166), ProcessCount=11 (was 11), AvailableMemoryMB=6117 (was 6131) 2024-11-24T04:26:02,832 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1264 is superior to 1024 2024-11-24T04:26:02,845 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayValueCompression#testCompactedBulkLoadedFiles Thread=430, OpenFileDescriptor=1264, MaxFileDescriptor=1048576, SystemLoadAverage=166, ProcessCount=11, AvailableMemoryMB=6117 2024-11-24T04:26:02,845 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1264 is superior to 1024 2024-11-24T04:26:02,860 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T04:26:02,862 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T04:26:02,863 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-24T04:26:02,865 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-70798231, suffix=, logDir=hdfs://localhost:44611/hbase/WALs/hregion-70798231, archiveDir=hdfs://localhost:44611/hbase/oldWALs, maxLogs=32 2024-11-24T04:26:02,878 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-70798231/hregion-70798231.1732422362865, exclude list is [], retry=0 2024-11-24T04:26:02,880 DEBUG [AsyncFSWAL-26-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44781,DS-54254ac8-0fae-43f7-ab06-53faa0e05dd7,DISK] 2024-11-24T04:26:02,881 DEBUG [AsyncFSWAL-26-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36069,DS-11c8f594-4fc2-47e4-b222-092ec50eb991,DISK] 2024-11-24T04:26:02,881 DEBUG [AsyncFSWAL-26-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34457,DS-20bb5930-23a3-441d-bdb2-015c0d56b2ec,DISK] 2024-11-24T04:26:02,882 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-70798231/hregion-70798231.1732422362865 2024-11-24T04:26:02,883 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35691:35691),(127.0.0.1/127.0.0.1:41009:41009),(127.0.0.1/127.0.0.1:33347:33347)] 2024-11-24T04:26:02,883 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => ee71304fdc561957e261401aa77df1c5, NAME => 'testCompactedBulkLoadedFiles,,1732422362861.ee71304fdc561957e261401aa77df1c5.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testCompactedBulkLoadedFiles', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44611/hbase 2024-11-24T04:26:02,891 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44781 is added to blk_1073741916_1094 (size=63) 2024-11-24T04:26:02,891 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34457 is added to blk_1073741916_1094 (size=63) 2024-11-24T04:26:02,891 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36069 is added to blk_1073741916_1094 (size=63) 2024-11-24T04:26:02,892 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testCompactedBulkLoadedFiles,,1732422362861.ee71304fdc561957e261401aa77df1c5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T04:26:02,893 INFO [StoreOpener-ee71304fdc561957e261401aa77df1c5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region ee71304fdc561957e261401aa77df1c5 2024-11-24T04:26:02,894 INFO [StoreOpener-ee71304fdc561957e261401aa77df1c5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ee71304fdc561957e261401aa77df1c5 columnFamilyName a 2024-11-24T04:26:02,894 DEBUG [StoreOpener-ee71304fdc561957e261401aa77df1c5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:26:02,895 INFO [StoreOpener-ee71304fdc561957e261401aa77df1c5-1 {}] regionserver.HStore(327): Store=ee71304fdc561957e261401aa77df1c5/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:26:02,895 INFO [StoreOpener-ee71304fdc561957e261401aa77df1c5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region ee71304fdc561957e261401aa77df1c5 2024-11-24T04:26:02,896 INFO [StoreOpener-ee71304fdc561957e261401aa77df1c5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ee71304fdc561957e261401aa77df1c5 columnFamilyName b 2024-11-24T04:26:02,896 DEBUG [StoreOpener-ee71304fdc561957e261401aa77df1c5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:26:02,896 INFO [StoreOpener-ee71304fdc561957e261401aa77df1c5-1 {}] regionserver.HStore(327): Store=ee71304fdc561957e261401aa77df1c5/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:26:02,896 INFO [StoreOpener-ee71304fdc561957e261401aa77df1c5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region ee71304fdc561957e261401aa77df1c5 2024-11-24T04:26:02,898 INFO [StoreOpener-ee71304fdc561957e261401aa77df1c5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ee71304fdc561957e261401aa77df1c5 columnFamilyName c 2024-11-24T04:26:02,898 DEBUG [StoreOpener-ee71304fdc561957e261401aa77df1c5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:26:02,898 INFO [StoreOpener-ee71304fdc561957e261401aa77df1c5-1 {}] regionserver.HStore(327): Store=ee71304fdc561957e261401aa77df1c5/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:26:02,898 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for ee71304fdc561957e261401aa77df1c5 2024-11-24T04:26:02,899 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44611/hbase/data/default/testCompactedBulkLoadedFiles/ee71304fdc561957e261401aa77df1c5 2024-11-24T04:26:02,900 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44611/hbase/data/default/testCompactedBulkLoadedFiles/ee71304fdc561957e261401aa77df1c5 2024-11-24T04:26:02,901 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for ee71304fdc561957e261401aa77df1c5 2024-11-24T04:26:02,901 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for ee71304fdc561957e261401aa77df1c5 2024-11-24T04:26:02,902 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testCompactedBulkLoadedFiles descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-24T04:26:02,903 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for ee71304fdc561957e261401aa77df1c5 2024-11-24T04:26:02,906 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44611/hbase/data/default/testCompactedBulkLoadedFiles/ee71304fdc561957e261401aa77df1c5/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T04:26:02,906 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened ee71304fdc561957e261401aa77df1c5; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59409147, jitterRate=-0.1147347241640091}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-24T04:26:02,907 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for ee71304fdc561957e261401aa77df1c5: Writing region info on filesystem at 1732422362892Initializing all the Stores at 1732422362893 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732422362893Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732422362893Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732422362893Cleaning up temporary data from old regions at 1732422362901 (+8 ms)Region opened successfully at 1732422362907 (+6 ms) 2024-11-24T04:26:02,907 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing ee71304fdc561957e261401aa77df1c5, disabling compactions & flushes 2024-11-24T04:26:02,907 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testCompactedBulkLoadedFiles,,1732422362861.ee71304fdc561957e261401aa77df1c5. 2024-11-24T04:26:02,907 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testCompactedBulkLoadedFiles,,1732422362861.ee71304fdc561957e261401aa77df1c5. 2024-11-24T04:26:02,907 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testCompactedBulkLoadedFiles,,1732422362861.ee71304fdc561957e261401aa77df1c5. after waiting 0 ms 2024-11-24T04:26:02,907 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testCompactedBulkLoadedFiles,,1732422362861.ee71304fdc561957e261401aa77df1c5. 2024-11-24T04:26:02,907 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testCompactedBulkLoadedFiles,,1732422362861.ee71304fdc561957e261401aa77df1c5. 2024-11-24T04:26:02,907 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for ee71304fdc561957e261401aa77df1c5: Waiting for close lock at 1732422362907Disabling compacts and flushes for region at 1732422362907Disabling writes for close at 1732422362907Writing region close event to WAL at 1732422362907Closed at 1732422362907 2024-11-24T04:26:02,911 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36069 is added to blk_1073741915_1093 (size=95) 2024-11-24T04:26:02,911 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34457 is added to blk_1073741915_1093 (size=95) 2024-11-24T04:26:02,911 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44781 is added to blk_1073741915_1093 (size=95) 2024-11-24T04:26:02,915 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-24T04:26:02,915 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-70798231:(num 1732422362865) 2024-11-24T04:26:02,915 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-24T04:26:02,917 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:44611/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732422362860, archiveDir=hdfs://localhost:44611/hbase/oldWALs, maxLogs=32 2024-11-24T04:26:02,931 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732422362860/wal.1732422362917, exclude list is [], retry=0 2024-11-24T04:26:02,933 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44781,DS-54254ac8-0fae-43f7-ab06-53faa0e05dd7,DISK] 2024-11-24T04:26:02,934 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36069,DS-11c8f594-4fc2-47e4-b222-092ec50eb991,DISK] 2024-11-24T04:26:02,934 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34457,DS-20bb5930-23a3-441d-bdb2-015c0d56b2ec,DISK] 2024-11-24T04:26:02,936 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732422362860/wal.1732422362917 2024-11-24T04:26:02,936 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35691:35691),(127.0.0.1/127.0.0.1:41009:41009),(127.0.0.1/127.0.0.1:33347:33347)] 2024-11-24T04:26:02,936 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => ee71304fdc561957e261401aa77df1c5, NAME => 'testCompactedBulkLoadedFiles,,1732422362861.ee71304fdc561957e261401aa77df1c5.', STARTKEY => '', ENDKEY => ''} 2024-11-24T04:26:02,936 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testCompactedBulkLoadedFiles,,1732422362861.ee71304fdc561957e261401aa77df1c5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T04:26:02,936 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for ee71304fdc561957e261401aa77df1c5 2024-11-24T04:26:02,936 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for ee71304fdc561957e261401aa77df1c5 2024-11-24T04:26:02,938 INFO [StoreOpener-ee71304fdc561957e261401aa77df1c5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region ee71304fdc561957e261401aa77df1c5 2024-11-24T04:26:02,939 INFO [StoreOpener-ee71304fdc561957e261401aa77df1c5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ee71304fdc561957e261401aa77df1c5 columnFamilyName a 2024-11-24T04:26:02,939 DEBUG [StoreOpener-ee71304fdc561957e261401aa77df1c5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:26:02,939 INFO [StoreOpener-ee71304fdc561957e261401aa77df1c5-1 {}] regionserver.HStore(327): Store=ee71304fdc561957e261401aa77df1c5/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:26:02,940 INFO [StoreOpener-ee71304fdc561957e261401aa77df1c5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region ee71304fdc561957e261401aa77df1c5 2024-11-24T04:26:02,940 INFO [StoreOpener-ee71304fdc561957e261401aa77df1c5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ee71304fdc561957e261401aa77df1c5 columnFamilyName b 2024-11-24T04:26:02,940 DEBUG [StoreOpener-ee71304fdc561957e261401aa77df1c5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:26:02,941 INFO [StoreOpener-ee71304fdc561957e261401aa77df1c5-1 {}] regionserver.HStore(327): Store=ee71304fdc561957e261401aa77df1c5/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:26:02,941 INFO [StoreOpener-ee71304fdc561957e261401aa77df1c5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region ee71304fdc561957e261401aa77df1c5 2024-11-24T04:26:02,942 INFO [StoreOpener-ee71304fdc561957e261401aa77df1c5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ee71304fdc561957e261401aa77df1c5 columnFamilyName c 2024-11-24T04:26:02,942 DEBUG [StoreOpener-ee71304fdc561957e261401aa77df1c5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:26:02,942 INFO [StoreOpener-ee71304fdc561957e261401aa77df1c5-1 {}] regionserver.HStore(327): Store=ee71304fdc561957e261401aa77df1c5/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:26:02,942 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for ee71304fdc561957e261401aa77df1c5 2024-11-24T04:26:02,943 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44611/hbase/data/default/testCompactedBulkLoadedFiles/ee71304fdc561957e261401aa77df1c5 2024-11-24T04:26:02,944 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44611/hbase/data/default/testCompactedBulkLoadedFiles/ee71304fdc561957e261401aa77df1c5 2024-11-24T04:26:02,945 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for ee71304fdc561957e261401aa77df1c5 2024-11-24T04:26:02,945 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for ee71304fdc561957e261401aa77df1c5 2024-11-24T04:26:02,946 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testCompactedBulkLoadedFiles descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-24T04:26:02,947 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for ee71304fdc561957e261401aa77df1c5 2024-11-24T04:26:02,948 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened ee71304fdc561957e261401aa77df1c5; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65894384, jitterRate=-0.0180971622467041}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-24T04:26:02,949 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for ee71304fdc561957e261401aa77df1c5: Writing region info on filesystem at 1732422362937Initializing all the Stores at 1732422362937Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732422362938 (+1 ms)Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732422362938Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732422362938Cleaning up temporary data from old regions at 1732422362945 (+7 ms)Region opened successfully at 1732422362949 (+4 ms) 2024-11-24T04:26:02,953 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44611/hbase/testCompactedBulkLoadedFiles/hfile0 is 32, key is 000/a:a/1732422362952/Put/seqid=0 2024-11-24T04:26:02,962 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34457 is added to blk_1073741918_1096 (size=4875) 2024-11-24T04:26:02,962 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44781 is added to blk_1073741918_1096 (size=4875) 2024-11-24T04:26:02,963 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36069 is added to blk_1073741918_1096 (size=4875) 2024-11-24T04:26:02,966 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44611/hbase/testCompactedBulkLoadedFiles/hfile1 is 32, key is 100/a:a/1732422362966/Put/seqid=0 2024-11-24T04:26:02,974 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36069 is added to blk_1073741919_1097 (size=4875) 2024-11-24T04:26:02,974 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44781 is added to blk_1073741919_1097 (size=4875) 2024-11-24T04:26:02,977 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34457 is added to blk_1073741919_1097 (size=4875) 2024-11-24T04:26:02,979 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44611/hbase/testCompactedBulkLoadedFiles/hfile2 is 32, key is 200/a:a/1732422362979/Put/seqid=0 2024-11-24T04:26:02,986 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36069 is added to blk_1073741920_1098 (size=4875) 2024-11-24T04:26:02,986 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44781 is added to blk_1073741920_1098 (size=4875) 2024-11-24T04:26:02,987 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34457 is added to blk_1073741920_1098 (size=4875) 2024-11-24T04:26:02,987 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:44611/hbase/testCompactedBulkLoadedFiles/hfile0 for inclusion in ee71304fdc561957e261401aa77df1c5/a 2024-11-24T04:26:02,991 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first=000 last=050 2024-11-24T04:26:02,991 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-11-24T04:26:02,991 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:44611/hbase/testCompactedBulkLoadedFiles/hfile1 for inclusion in ee71304fdc561957e261401aa77df1c5/a 2024-11-24T04:26:02,995 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first=100 last=150 2024-11-24T04:26:02,995 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-11-24T04:26:02,995 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:44611/hbase/testCompactedBulkLoadedFiles/hfile2 for inclusion in ee71304fdc561957e261401aa77df1c5/a 2024-11-24T04:26:02,998 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first=200 last=250 2024-11-24T04:26:02,998 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-11-24T04:26:02,998 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing ee71304fdc561957e261401aa77df1c5 3/3 column families, dataSize=51 B heapSize=896 B 2024-11-24T04:26:03,014 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44611/hbase/data/default/testCompactedBulkLoadedFiles/ee71304fdc561957e261401aa77df1c5/.tmp/a/b304da38e31049e0a8f5bc6016fd4507 is 55, key is testCompactedBulkLoadedFiles/a:a/1732422362949/Put/seqid=0 2024-11-24T04:26:03,021 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36069 is added to blk_1073741921_1099 (size=5107) 2024-11-24T04:26:03,021 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34457 is added to blk_1073741921_1099 (size=5107) 2024-11-24T04:26:03,022 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44781 is added to blk_1073741921_1099 (size=5107) 2024-11-24T04:26:03,022 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51 B at sequenceid=4 (bloomFilter=true), to=hdfs://localhost:44611/hbase/data/default/testCompactedBulkLoadedFiles/ee71304fdc561957e261401aa77df1c5/.tmp/a/b304da38e31049e0a8f5bc6016fd4507 2024-11-24T04:26:03,029 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44611/hbase/data/default/testCompactedBulkLoadedFiles/ee71304fdc561957e261401aa77df1c5/.tmp/a/b304da38e31049e0a8f5bc6016fd4507 as hdfs://localhost:44611/hbase/data/default/testCompactedBulkLoadedFiles/ee71304fdc561957e261401aa77df1c5/a/b304da38e31049e0a8f5bc6016fd4507 2024-11-24T04:26:03,034 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44611/hbase/data/default/testCompactedBulkLoadedFiles/ee71304fdc561957e261401aa77df1c5/a/b304da38e31049e0a8f5bc6016fd4507, entries=1, sequenceid=4, filesize=5.0 K 2024-11-24T04:26:03,036 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~51 B/51, heapSize ~368 B/368, currentSize=0 B/0 for ee71304fdc561957e261401aa77df1c5 in 38ms, sequenceid=4, compaction requested=false 2024-11-24T04:26:03,036 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for ee71304fdc561957e261401aa77df1c5: 2024-11-24T04:26:03,038 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44611/hbase/testCompactedBulkLoadedFiles/hfile0 as hdfs://localhost:44611/hbase/data/default/testCompactedBulkLoadedFiles/ee71304fdc561957e261401aa77df1c5/a/ff200046a69846f1a6cb5eb815e9c251_SeqId_4_ 2024-11-24T04:26:03,039 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44611/hbase/testCompactedBulkLoadedFiles/hfile1 as hdfs://localhost:44611/hbase/data/default/testCompactedBulkLoadedFiles/ee71304fdc561957e261401aa77df1c5/a/62c6b1bf5abe4dc4b3280ba5d2d279c6_SeqId_4_ 2024-11-24T04:26:03,040 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44611/hbase/testCompactedBulkLoadedFiles/hfile2 as hdfs://localhost:44611/hbase/data/default/testCompactedBulkLoadedFiles/ee71304fdc561957e261401aa77df1c5/a/e3cd676e7eab4bd4aa536330b6a06f49_SeqId_4_ 2024-11-24T04:26:03,040 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:44611/hbase/testCompactedBulkLoadedFiles/hfile0 into ee71304fdc561957e261401aa77df1c5/a as hdfs://localhost:44611/hbase/data/default/testCompactedBulkLoadedFiles/ee71304fdc561957e261401aa77df1c5/a/ff200046a69846f1a6cb5eb815e9c251_SeqId_4_ - updating store file list. 2024-11-24T04:26:03,045 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for ff200046a69846f1a6cb5eb815e9c251_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-24T04:26:03,045 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:44611/hbase/data/default/testCompactedBulkLoadedFiles/ee71304fdc561957e261401aa77df1c5/a/ff200046a69846f1a6cb5eb815e9c251_SeqId_4_ into ee71304fdc561957e261401aa77df1c5/a 2024-11-24T04:26:03,045 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:44611/hbase/testCompactedBulkLoadedFiles/hfile0 into ee71304fdc561957e261401aa77df1c5/a (new location: hdfs://localhost:44611/hbase/data/default/testCompactedBulkLoadedFiles/ee71304fdc561957e261401aa77df1c5/a/ff200046a69846f1a6cb5eb815e9c251_SeqId_4_) 2024-11-24T04:26:03,046 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:44611/hbase/testCompactedBulkLoadedFiles/hfile1 into ee71304fdc561957e261401aa77df1c5/a as hdfs://localhost:44611/hbase/data/default/testCompactedBulkLoadedFiles/ee71304fdc561957e261401aa77df1c5/a/62c6b1bf5abe4dc4b3280ba5d2d279c6_SeqId_4_ - updating store file list. 2024-11-24T04:26:03,051 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for 62c6b1bf5abe4dc4b3280ba5d2d279c6_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-24T04:26:03,051 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:44611/hbase/data/default/testCompactedBulkLoadedFiles/ee71304fdc561957e261401aa77df1c5/a/62c6b1bf5abe4dc4b3280ba5d2d279c6_SeqId_4_ into ee71304fdc561957e261401aa77df1c5/a 2024-11-24T04:26:03,051 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:44611/hbase/testCompactedBulkLoadedFiles/hfile1 into ee71304fdc561957e261401aa77df1c5/a (new location: hdfs://localhost:44611/hbase/data/default/testCompactedBulkLoadedFiles/ee71304fdc561957e261401aa77df1c5/a/62c6b1bf5abe4dc4b3280ba5d2d279c6_SeqId_4_) 2024-11-24T04:26:03,052 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:44611/hbase/testCompactedBulkLoadedFiles/hfile2 into ee71304fdc561957e261401aa77df1c5/a as hdfs://localhost:44611/hbase/data/default/testCompactedBulkLoadedFiles/ee71304fdc561957e261401aa77df1c5/a/e3cd676e7eab4bd4aa536330b6a06f49_SeqId_4_ - updating store file list. 2024-11-24T04:26:03,057 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for e3cd676e7eab4bd4aa536330b6a06f49_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-24T04:26:03,058 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:44611/hbase/data/default/testCompactedBulkLoadedFiles/ee71304fdc561957e261401aa77df1c5/a/e3cd676e7eab4bd4aa536330b6a06f49_SeqId_4_ into ee71304fdc561957e261401aa77df1c5/a 2024-11-24T04:26:03,058 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:44611/hbase/testCompactedBulkLoadedFiles/hfile2 into ee71304fdc561957e261401aa77df1c5/a (new location: hdfs://localhost:44611/hbase/data/default/testCompactedBulkLoadedFiles/ee71304fdc561957e261401aa77df1c5/a/e3cd676e7eab4bd4aa536330b6a06f49_SeqId_4_) 2024-11-24T04:26:03,066 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T04:26:03,066 DEBUG [Time-limited test {}] regionserver.HStore(1541): ee71304fdc561957e261401aa77df1c5/a is initiating major compaction (all files) 2024-11-24T04:26:03,066 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of ee71304fdc561957e261401aa77df1c5/a in testCompactedBulkLoadedFiles,,1732422362861.ee71304fdc561957e261401aa77df1c5. 2024-11-24T04:26:03,066 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44611/hbase/data/default/testCompactedBulkLoadedFiles/ee71304fdc561957e261401aa77df1c5/a/b304da38e31049e0a8f5bc6016fd4507, hdfs://localhost:44611/hbase/data/default/testCompactedBulkLoadedFiles/ee71304fdc561957e261401aa77df1c5/a/ff200046a69846f1a6cb5eb815e9c251_SeqId_4_, hdfs://localhost:44611/hbase/data/default/testCompactedBulkLoadedFiles/ee71304fdc561957e261401aa77df1c5/a/62c6b1bf5abe4dc4b3280ba5d2d279c6_SeqId_4_, hdfs://localhost:44611/hbase/data/default/testCompactedBulkLoadedFiles/ee71304fdc561957e261401aa77df1c5/a/e3cd676e7eab4bd4aa536330b6a06f49_SeqId_4_] into tmpdir=hdfs://localhost:44611/hbase/data/default/testCompactedBulkLoadedFiles/ee71304fdc561957e261401aa77df1c5/.tmp, totalSize=19.3 K 2024-11-24T04:26:03,067 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting b304da38e31049e0a8f5bc6016fd4507, keycount=1, bloomtype=ROW, size=5.0 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=1732422362949 2024-11-24T04:26:03,067 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting ff200046a69846f1a6cb5eb815e9c251_SeqId_4_, keycount=10, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=-9223372036854775808 2024-11-24T04:26:03,067 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 62c6b1bf5abe4dc4b3280ba5d2d279c6_SeqId_4_, keycount=10, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=-9223372036854775808 2024-11-24T04:26:03,068 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting e3cd676e7eab4bd4aa536330b6a06f49_SeqId_4_, keycount=10, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=-9223372036854775808 2024-11-24T04:26:03,084 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44611/hbase/data/default/testCompactedBulkLoadedFiles/ee71304fdc561957e261401aa77df1c5/.tmp/a/13c7bdd6ff164cbcb4e3d558f796dea5 is 55, key is testCompactedBulkLoadedFiles/a:a/1732422362949/Put/seqid=0 2024-11-24T04:26:03,098 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34457 is added to blk_1073741922_1100 (size=6154) 2024-11-24T04:26:03,104 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36069 is added to blk_1073741922_1100 (size=6154) 2024-11-24T04:26:03,104 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44781 is added to blk_1073741922_1100 (size=6154) 2024-11-24T04:26:03,107 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44611/hbase/data/default/testCompactedBulkLoadedFiles/ee71304fdc561957e261401aa77df1c5/.tmp/a/13c7bdd6ff164cbcb4e3d558f796dea5 as hdfs://localhost:44611/hbase/data/default/testCompactedBulkLoadedFiles/ee71304fdc561957e261401aa77df1c5/a/13c7bdd6ff164cbcb4e3d558f796dea5 2024-11-24T04:26:03,114 INFO [Time-limited test {}] regionserver.HStore(1337): Completed major compaction of 4 (all) file(s) in ee71304fdc561957e261401aa77df1c5/a of ee71304fdc561957e261401aa77df1c5 into 13c7bdd6ff164cbcb4e3d558f796dea5(size=6.0 K), total size for store is 6.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T04:26:03,114 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for ee71304fdc561957e261401aa77df1c5: 2024-11-24T04:26:03,114 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 0 store files, 0 compacting, 0 eligible, 16 blocking 2024-11-24T04:26:03,114 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 0 store files, 0 compacting, 0 eligible, 16 blocking 2024-11-24T04:26:03,161 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:44611/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732422362860/wal.1732422362917, size=0 (0bytes) 2024-11-24T04:26:03,161 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:44611/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732422362860/wal.1732422362917 might be still open, length is 0 2024-11-24T04:26:03,161 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:44611/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732422362860/wal.1732422362917 2024-11-24T04:26:03,162 WARN [IPC Server handler 2 on default port 44611 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732422362860/wal.1732422362917 has not been closed. Lease recovery is in progress. RecoveryId = 1101 for block blk_1073741917_1095 2024-11-24T04:26:03,162 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:44611/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732422362860/wal.1732422362917 after 1ms 2024-11-24T04:26:03,309 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testDatalossWhenInputError 2024-11-24T04:26:03,309 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testDatalossWhenInputError Metrics about Tables on a single HBase RegionServer 2024-11-24T04:26:03,310 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testCompactedBulkLoadedFiles 2024-11-24T04:26:03,310 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testCompactedBulkLoadedFiles Metrics about Tables on a single HBase RegionServer 2024-11-24T04:26:04,836 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1575425110_22 at /127.0.0.1:55360 [Receiving block BP-1091946017-172.17.0.2-1732422329789:blk_1073741917_1095] {}] datanode.DataXceiver(331): 127.0.0.1:44781:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:55360 dst: /127.0.0.1:44781 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:44781 remote=/127.0.0.1:55360]. Total timeout mills is 60000, 58278 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T04:26:04,837 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1575425110_22 at /127.0.0.1:45986 [Receiving block BP-1091946017-172.17.0.2-1732422329789:blk_1073741917_1095] {}] datanode.DataXceiver(331): 127.0.0.1:34457:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:45986 dst: /127.0.0.1:34457 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T04:26:04,837 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1575425110_22 at /127.0.0.1:58584 [Receiving block BP-1091946017-172.17.0.2-1732422329789:blk_1073741917_1095] {}] datanode.DataXceiver(331): 127.0.0.1:36069:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:58584 dst: /127.0.0.1:36069 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T04:26:04,842 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44781 is added to blk_1073741917_1101 (size=1170) 2024-11-24T04:26:04,842 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36069 is added to blk_1073741917_1101 (size=1170) 2024-11-24T04:26:04,842 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34457 is added to blk_1073741917_1101 (size=1170) 2024-11-24T04:26:07,163 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:44611/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732422362860/wal.1732422362917 after 4002ms 2024-11-24T04:26:07,166 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:44611/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732422362860/wal.1732422362917: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-24T04:26:07,166 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:44611/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732422362860/wal.1732422362917 took 4005ms 2024-11-24T04:26:07,168 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:44611/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732422362860/wal.1732422362917; continuing. 2024-11-24T04:26:07,168 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:44611/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732422362860/wal.1732422362917 so closing down 2024-11-24T04:26:07,168 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-24T04:26:07,169 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1732422362917.temp 2024-11-24T04:26:07,171 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:44611/hbase/data/default/testCompactedBulkLoadedFiles/ee71304fdc561957e261401aa77df1c5/recovered.edits/0000000000000000003-wal.1732422362917.temp 2024-11-24T04:26:07,171 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-24T04:26:07,176 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36069 is added to blk_1073741923_1102 (size=546) 2024-11-24T04:26:07,177 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34457 is added to blk_1073741923_1102 (size=546) 2024-11-24T04:26:07,177 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44781 is added to blk_1073741923_1102 (size=546) 2024-11-24T04:26:07,178 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:44611/hbase/data/default/testCompactedBulkLoadedFiles/ee71304fdc561957e261401aa77df1c5/recovered.edits/0000000000000000003-wal.1732422362917.temp (wrote 2 edits, skipped 0 edits in 0 ms) 2024-11-24T04:26:07,179 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:44611/hbase/data/default/testCompactedBulkLoadedFiles/ee71304fdc561957e261401aa77df1c5/recovered.edits/0000000000000000003-wal.1732422362917.temp to hdfs://localhost:44611/hbase/data/default/testCompactedBulkLoadedFiles/ee71304fdc561957e261401aa77df1c5/recovered.edits/0000000000000000008 2024-11-24T04:26:07,179 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 5 edits across 1 Regions in 13 ms; skipped=3; WAL=hdfs://localhost:44611/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732422362860/wal.1732422362917, size=0, length=0, corrupted=false, cancelled=false 2024-11-24T04:26:07,179 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:44611/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732422362860/wal.1732422362917, journal: Splitting hdfs://localhost:44611/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732422362860/wal.1732422362917, size=0 (0bytes) at 1732422363161Finishing writing output for hdfs://localhost:44611/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732422362860/wal.1732422362917 so closing down at 1732422367168 (+4007 ms)Creating recovered edits writer path=hdfs://localhost:44611/hbase/data/default/testCompactedBulkLoadedFiles/ee71304fdc561957e261401aa77df1c5/recovered.edits/0000000000000000003-wal.1732422362917.temp at 1732422367171 (+3 ms)3 split writer threads finished at 1732422367171Closed recovered edits writer path=hdfs://localhost:44611/hbase/data/default/testCompactedBulkLoadedFiles/ee71304fdc561957e261401aa77df1c5/recovered.edits/0000000000000000003-wal.1732422362917.temp (wrote 2 edits, skipped 0 edits in 0 ms) at 1732422367178 (+7 ms)Rename recovered edits hdfs://localhost:44611/hbase/data/default/testCompactedBulkLoadedFiles/ee71304fdc561957e261401aa77df1c5/recovered.edits/0000000000000000003-wal.1732422362917.temp to hdfs://localhost:44611/hbase/data/default/testCompactedBulkLoadedFiles/ee71304fdc561957e261401aa77df1c5/recovered.edits/0000000000000000008 at 1732422367179 (+1 ms)Processed 5 edits across 1 Regions in 13 ms; skipped=3; WAL=hdfs://localhost:44611/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732422362860/wal.1732422362917, size=0, length=0, corrupted=false, cancelled=false at 1732422367179 2024-11-24T04:26:07,180 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:44611/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732422362860/wal.1732422362917 to hdfs://localhost:44611/hbase/oldWALs/wal.1732422362917 2024-11-24T04:26:07,181 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:44611/hbase/data/default/testCompactedBulkLoadedFiles/ee71304fdc561957e261401aa77df1c5/recovered.edits/0000000000000000008 2024-11-24T04:26:07,181 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-24T04:26:07,183 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:44611/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732422362860, archiveDir=hdfs://localhost:44611/hbase/oldWALs, maxLogs=32 2024-11-24T04:26:07,196 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732422362860/wal.1732422367183, exclude list is [], retry=0 2024-11-24T04:26:07,198 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36069,DS-11c8f594-4fc2-47e4-b222-092ec50eb991,DISK] 2024-11-24T04:26:07,199 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34457,DS-20bb5930-23a3-441d-bdb2-015c0d56b2ec,DISK] 2024-11-24T04:26:07,199 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44781,DS-54254ac8-0fae-43f7-ab06-53faa0e05dd7,DISK] 2024-11-24T04:26:07,200 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732422362860/wal.1732422367183 2024-11-24T04:26:07,201 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41009:41009),(127.0.0.1/127.0.0.1:33347:33347),(127.0.0.1/127.0.0.1:35691:35691)] 2024-11-24T04:26:07,201 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => ee71304fdc561957e261401aa77df1c5, NAME => 'testCompactedBulkLoadedFiles,,1732422362861.ee71304fdc561957e261401aa77df1c5.', STARTKEY => '', ENDKEY => ''} 2024-11-24T04:26:07,201 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testCompactedBulkLoadedFiles,,1732422362861.ee71304fdc561957e261401aa77df1c5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T04:26:07,201 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for ee71304fdc561957e261401aa77df1c5 2024-11-24T04:26:07,201 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for ee71304fdc561957e261401aa77df1c5 2024-11-24T04:26:07,202 INFO [StoreOpener-ee71304fdc561957e261401aa77df1c5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region ee71304fdc561957e261401aa77df1c5 2024-11-24T04:26:07,203 INFO [StoreOpener-ee71304fdc561957e261401aa77df1c5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ee71304fdc561957e261401aa77df1c5 columnFamilyName a 2024-11-24T04:26:07,203 DEBUG [StoreOpener-ee71304fdc561957e261401aa77df1c5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:26:07,209 DEBUG [StoreOpener-ee71304fdc561957e261401aa77df1c5-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:44611/hbase/data/default/testCompactedBulkLoadedFiles/ee71304fdc561957e261401aa77df1c5/a/13c7bdd6ff164cbcb4e3d558f796dea5 2024-11-24T04:26:07,213 DEBUG [StoreFileOpener-ee71304fdc561957e261401aa77df1c5-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for 62c6b1bf5abe4dc4b3280ba5d2d279c6_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-24T04:26:07,213 DEBUG [StoreOpener-ee71304fdc561957e261401aa77df1c5-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:44611/hbase/data/default/testCompactedBulkLoadedFiles/ee71304fdc561957e261401aa77df1c5/a/62c6b1bf5abe4dc4b3280ba5d2d279c6_SeqId_4_ 2024-11-24T04:26:07,216 DEBUG [StoreOpener-ee71304fdc561957e261401aa77df1c5-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:44611/hbase/data/default/testCompactedBulkLoadedFiles/ee71304fdc561957e261401aa77df1c5/a/b304da38e31049e0a8f5bc6016fd4507 2024-11-24T04:26:07,220 DEBUG [StoreFileOpener-ee71304fdc561957e261401aa77df1c5-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for e3cd676e7eab4bd4aa536330b6a06f49_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-24T04:26:07,220 DEBUG [StoreOpener-ee71304fdc561957e261401aa77df1c5-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:44611/hbase/data/default/testCompactedBulkLoadedFiles/ee71304fdc561957e261401aa77df1c5/a/e3cd676e7eab4bd4aa536330b6a06f49_SeqId_4_ 2024-11-24T04:26:07,223 DEBUG [StoreFileOpener-ee71304fdc561957e261401aa77df1c5-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for ff200046a69846f1a6cb5eb815e9c251_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-24T04:26:07,223 DEBUG [StoreOpener-ee71304fdc561957e261401aa77df1c5-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:44611/hbase/data/default/testCompactedBulkLoadedFiles/ee71304fdc561957e261401aa77df1c5/a/ff200046a69846f1a6cb5eb815e9c251_SeqId_4_ 2024-11-24T04:26:07,223 WARN [StoreOpener-ee71304fdc561957e261401aa77df1c5-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:44611/hbase/data/default/testCompactedBulkLoadedFiles/ee71304fdc561957e261401aa77df1c5/a/62c6b1bf5abe4dc4b3280ba5d2d279c6_SeqId_4_ from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@692ecc45 2024-11-24T04:26:07,223 WARN [StoreOpener-ee71304fdc561957e261401aa77df1c5-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:44611/hbase/data/default/testCompactedBulkLoadedFiles/ee71304fdc561957e261401aa77df1c5/a/b304da38e31049e0a8f5bc6016fd4507 from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@692ecc45 2024-11-24T04:26:07,223 WARN [StoreOpener-ee71304fdc561957e261401aa77df1c5-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:44611/hbase/data/default/testCompactedBulkLoadedFiles/ee71304fdc561957e261401aa77df1c5/a/e3cd676e7eab4bd4aa536330b6a06f49_SeqId_4_ from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@692ecc45 2024-11-24T04:26:07,223 WARN [StoreOpener-ee71304fdc561957e261401aa77df1c5-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:44611/hbase/data/default/testCompactedBulkLoadedFiles/ee71304fdc561957e261401aa77df1c5/a/ff200046a69846f1a6cb5eb815e9c251_SeqId_4_ from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@692ecc45 2024-11-24T04:26:07,223 DEBUG [StoreOpener-ee71304fdc561957e261401aa77df1c5-1 {}] regionserver.StoreEngine(327): Moving the files [hdfs://localhost:44611/hbase/data/default/testCompactedBulkLoadedFiles/ee71304fdc561957e261401aa77df1c5/a/62c6b1bf5abe4dc4b3280ba5d2d279c6_SeqId_4_, hdfs://localhost:44611/hbase/data/default/testCompactedBulkLoadedFiles/ee71304fdc561957e261401aa77df1c5/a/b304da38e31049e0a8f5bc6016fd4507, hdfs://localhost:44611/hbase/data/default/testCompactedBulkLoadedFiles/ee71304fdc561957e261401aa77df1c5/a/e3cd676e7eab4bd4aa536330b6a06f49_SeqId_4_, hdfs://localhost:44611/hbase/data/default/testCompactedBulkLoadedFiles/ee71304fdc561957e261401aa77df1c5/a/ff200046a69846f1a6cb5eb815e9c251_SeqId_4_] to archive 2024-11-24T04:26:07,224 DEBUG [StoreOpener-ee71304fdc561957e261401aa77df1c5-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-24T04:26:07,226 DEBUG [StoreOpener-ee71304fdc561957e261401aa77df1c5-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44611/hbase/data/default/testCompactedBulkLoadedFiles/ee71304fdc561957e261401aa77df1c5/a/62c6b1bf5abe4dc4b3280ba5d2d279c6_SeqId_4_ to hdfs://localhost:44611/hbase/archive/data/default/testCompactedBulkLoadedFiles/ee71304fdc561957e261401aa77df1c5/a/62c6b1bf5abe4dc4b3280ba5d2d279c6_SeqId_4_ 2024-11-24T04:26:07,227 DEBUG [StoreOpener-ee71304fdc561957e261401aa77df1c5-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44611/hbase/data/default/testCompactedBulkLoadedFiles/ee71304fdc561957e261401aa77df1c5/a/b304da38e31049e0a8f5bc6016fd4507 to hdfs://localhost:44611/hbase/archive/data/default/testCompactedBulkLoadedFiles/ee71304fdc561957e261401aa77df1c5/a/b304da38e31049e0a8f5bc6016fd4507 2024-11-24T04:26:07,228 DEBUG [StoreOpener-ee71304fdc561957e261401aa77df1c5-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44611/hbase/data/default/testCompactedBulkLoadedFiles/ee71304fdc561957e261401aa77df1c5/a/e3cd676e7eab4bd4aa536330b6a06f49_SeqId_4_ to hdfs://localhost:44611/hbase/archive/data/default/testCompactedBulkLoadedFiles/ee71304fdc561957e261401aa77df1c5/a/e3cd676e7eab4bd4aa536330b6a06f49_SeqId_4_ 2024-11-24T04:26:07,229 DEBUG [StoreOpener-ee71304fdc561957e261401aa77df1c5-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44611/hbase/data/default/testCompactedBulkLoadedFiles/ee71304fdc561957e261401aa77df1c5/a/ff200046a69846f1a6cb5eb815e9c251_SeqId_4_ to hdfs://localhost:44611/hbase/archive/data/default/testCompactedBulkLoadedFiles/ee71304fdc561957e261401aa77df1c5/a/ff200046a69846f1a6cb5eb815e9c251_SeqId_4_ 2024-11-24T04:26:07,229 INFO [StoreOpener-ee71304fdc561957e261401aa77df1c5-1 {}] regionserver.HStore(327): Store=ee71304fdc561957e261401aa77df1c5/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:26:07,229 INFO [StoreOpener-ee71304fdc561957e261401aa77df1c5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region ee71304fdc561957e261401aa77df1c5 2024-11-24T04:26:07,230 INFO [StoreOpener-ee71304fdc561957e261401aa77df1c5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ee71304fdc561957e261401aa77df1c5 columnFamilyName b 2024-11-24T04:26:07,230 DEBUG [StoreOpener-ee71304fdc561957e261401aa77df1c5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:26:07,231 INFO [StoreOpener-ee71304fdc561957e261401aa77df1c5-1 {}] regionserver.HStore(327): Store=ee71304fdc561957e261401aa77df1c5/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:26:07,231 INFO [StoreOpener-ee71304fdc561957e261401aa77df1c5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region ee71304fdc561957e261401aa77df1c5 2024-11-24T04:26:07,232 INFO [StoreOpener-ee71304fdc561957e261401aa77df1c5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ee71304fdc561957e261401aa77df1c5 columnFamilyName c 2024-11-24T04:26:07,232 DEBUG [StoreOpener-ee71304fdc561957e261401aa77df1c5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:26:07,232 INFO [StoreOpener-ee71304fdc561957e261401aa77df1c5-1 {}] regionserver.HStore(327): Store=ee71304fdc561957e261401aa77df1c5/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:26:07,232 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for ee71304fdc561957e261401aa77df1c5 2024-11-24T04:26:07,233 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44611/hbase/data/default/testCompactedBulkLoadedFiles/ee71304fdc561957e261401aa77df1c5 2024-11-24T04:26:07,234 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:44611/hbase/data/default/testCompactedBulkLoadedFiles/ee71304fdc561957e261401aa77df1c5 2024-11-24T04:26:07,235 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:44611/hbase/data/default/testCompactedBulkLoadedFiles/ee71304fdc561957e261401aa77df1c5/recovered.edits/0000000000000000008 2024-11-24T04:26:07,236 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:44611/hbase/data/default/testCompactedBulkLoadedFiles/ee71304fdc561957e261401aa77df1c5/recovered.edits/0000000000000000008: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-24T04:26:07,238 DEBUG [Time-limited test {}] regionserver.HRegion(5836): ee71304fdc561957e261401aa77df1c5 : Replaying compaction marker table_name: "testCompactedBulkLoadedFiles" encoded_region_name: "ee71304fdc561957e261401aa77df1c5" family_name: "a" compaction_input: "b304da38e31049e0a8f5bc6016fd4507" compaction_input: "ff200046a69846f1a6cb5eb815e9c251_SeqId_4_" compaction_input: "62c6b1bf5abe4dc4b3280ba5d2d279c6_SeqId_4_" compaction_input: "e3cd676e7eab4bd4aa536330b6a06f49_SeqId_4_" compaction_output: "13c7bdd6ff164cbcb4e3d558f796dea5" store_home_dir: "a" region_name: "testCompactedBulkLoadedFiles,,1732422362861.ee71304fdc561957e261401aa77df1c5." with seqId=9223372036854775807 and lastReplayedOpenRegionSeqId=-1 2024-11-24T04:26:07,238 DEBUG [Time-limited test {}] regionserver.HStore(1354): Completing compaction from the WAL marker 2024-11-24T04:26:07,238 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 0, skipped 2, firstSequenceIdInLog=3, maxSequenceIdInLog=8, path=hdfs://localhost:44611/hbase/data/default/testCompactedBulkLoadedFiles/ee71304fdc561957e261401aa77df1c5/recovered.edits/0000000000000000008 2024-11-24T04:26:07,239 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:44611/hbase/data/default/testCompactedBulkLoadedFiles/ee71304fdc561957e261401aa77df1c5/recovered.edits/0000000000000000008 2024-11-24T04:26:07,240 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for ee71304fdc561957e261401aa77df1c5 2024-11-24T04:26:07,240 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for ee71304fdc561957e261401aa77df1c5 2024-11-24T04:26:07,241 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testCompactedBulkLoadedFiles descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-24T04:26:07,242 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for ee71304fdc561957e261401aa77df1c5 2024-11-24T04:26:07,245 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44611/hbase/data/default/testCompactedBulkLoadedFiles/ee71304fdc561957e261401aa77df1c5/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-24T04:26:07,246 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened ee71304fdc561957e261401aa77df1c5; next sequenceid=9; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=58907954, jitterRate=-0.12220308184623718}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-24T04:26:07,246 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for ee71304fdc561957e261401aa77df1c5: Writing region info on filesystem at 1732422367201Initializing all the Stores at 1732422367202 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732422367202Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732422367202Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732422367202Cleaning up temporary data from old regions at 1732422367240 (+38 ms)Region opened successfully at 1732422367246 (+6 ms) 2024-11-24T04:26:07,249 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing ee71304fdc561957e261401aa77df1c5, disabling compactions & flushes 2024-11-24T04:26:07,249 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testCompactedBulkLoadedFiles,,1732422362861.ee71304fdc561957e261401aa77df1c5. 2024-11-24T04:26:07,249 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testCompactedBulkLoadedFiles,,1732422362861.ee71304fdc561957e261401aa77df1c5. 2024-11-24T04:26:07,249 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testCompactedBulkLoadedFiles,,1732422362861.ee71304fdc561957e261401aa77df1c5. after waiting 0 ms 2024-11-24T04:26:07,249 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testCompactedBulkLoadedFiles,,1732422362861.ee71304fdc561957e261401aa77df1c5. 2024-11-24T04:26:07,250 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testCompactedBulkLoadedFiles,,1732422362861.ee71304fdc561957e261401aa77df1c5. 2024-11-24T04:26:07,250 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for ee71304fdc561957e261401aa77df1c5: Waiting for close lock at 1732422367249Disabling compacts and flushes for region at 1732422367249Disabling writes for close at 1732422367249Writing region close event to WAL at 1732422367249Closed at 1732422367249 2024-11-24T04:26:07,252 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36069 is added to blk_1073741924_1103 (size=95) 2024-11-24T04:26:07,252 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44781 is added to blk_1073741924_1103 (size=95) 2024-11-24T04:26:07,252 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34457 is added to blk_1073741924_1103 (size=95) 2024-11-24T04:26:07,254 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-24T04:26:07,254 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1732422367183) 2024-11-24T04:26:07,271 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayValueCompression#testCompactedBulkLoadedFiles Thread=442 (was 430) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-493246260_22 at /127.0.0.1:55486 [Waiting for operation #11] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-26-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkinstestCompactedBulkLoadedFiles@localhost:44611 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-26-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-493246260_22 at /127.0.0.1:46034 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1319272049) connection to localhost/127.0.0.1:44611 from jenkinstestCompactedBulkLoadedFiles java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: AsyncFSWAL-26-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1346 (was 1264) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=161 (was 166), ProcessCount=11 (was 11), AvailableMemoryMB=6086 (was 6117) 2024-11-24T04:26:07,271 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1346 is superior to 1024 2024-11-24T04:26:07,286 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayValueCompression#testReplayEditsWrittenViaHRegion Thread=442, OpenFileDescriptor=1346, MaxFileDescriptor=1048576, SystemLoadAverage=161, ProcessCount=11, AvailableMemoryMB=6085 2024-11-24T04:26:07,286 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1346 is superior to 1024 2024-11-24T04:26:07,306 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T04:26:07,308 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T04:26:07,309 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-24T04:26:07,311 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-09282742, suffix=, logDir=hdfs://localhost:44611/hbase/WALs/hregion-09282742, archiveDir=hdfs://localhost:44611/hbase/oldWALs, maxLogs=32 2024-11-24T04:26:07,324 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-09282742/hregion-09282742.1732422367311, exclude list is [], retry=0 2024-11-24T04:26:07,326 DEBUG [AsyncFSWAL-28-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44781,DS-54254ac8-0fae-43f7-ab06-53faa0e05dd7,DISK] 2024-11-24T04:26:07,327 DEBUG [AsyncFSWAL-28-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36069,DS-11c8f594-4fc2-47e4-b222-092ec50eb991,DISK] 2024-11-24T04:26:07,327 DEBUG [AsyncFSWAL-28-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34457,DS-20bb5930-23a3-441d-bdb2-015c0d56b2ec,DISK] 2024-11-24T04:26:07,329 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-09282742/hregion-09282742.1732422367311 2024-11-24T04:26:07,329 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35691:35691),(127.0.0.1/127.0.0.1:41009:41009),(127.0.0.1/127.0.0.1:33347:33347)] 2024-11-24T04:26:07,330 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 192450b6e874b4a6ec88b914c80976de, NAME => 'testReplayEditsWrittenViaHRegion,,1732422367307.192450b6e874b4a6ec88b914c80976de.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenViaHRegion', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44611/hbase 2024-11-24T04:26:07,340 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34457 is added to blk_1073741926_1105 (size=67) 2024-11-24T04:26:07,340 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36069 is added to blk_1073741926_1105 (size=67) 2024-11-24T04:26:07,341 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44781 is added to blk_1073741926_1105 (size=67) 2024-11-24T04:26:07,341 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1732422367307.192450b6e874b4a6ec88b914c80976de.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T04:26:07,342 INFO [StoreOpener-192450b6e874b4a6ec88b914c80976de-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 192450b6e874b4a6ec88b914c80976de 2024-11-24T04:26:07,344 INFO [StoreOpener-192450b6e874b4a6ec88b914c80976de-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 192450b6e874b4a6ec88b914c80976de columnFamilyName a 2024-11-24T04:26:07,344 DEBUG [StoreOpener-192450b6e874b4a6ec88b914c80976de-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:26:07,344 INFO [StoreOpener-192450b6e874b4a6ec88b914c80976de-1 {}] regionserver.HStore(327): Store=192450b6e874b4a6ec88b914c80976de/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:26:07,344 INFO [StoreOpener-192450b6e874b4a6ec88b914c80976de-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 192450b6e874b4a6ec88b914c80976de 2024-11-24T04:26:07,345 INFO [StoreOpener-192450b6e874b4a6ec88b914c80976de-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 192450b6e874b4a6ec88b914c80976de columnFamilyName b 2024-11-24T04:26:07,346 DEBUG [StoreOpener-192450b6e874b4a6ec88b914c80976de-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:26:07,346 INFO [StoreOpener-192450b6e874b4a6ec88b914c80976de-1 {}] regionserver.HStore(327): Store=192450b6e874b4a6ec88b914c80976de/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:26:07,346 INFO [StoreOpener-192450b6e874b4a6ec88b914c80976de-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 192450b6e874b4a6ec88b914c80976de 2024-11-24T04:26:07,347 INFO [StoreOpener-192450b6e874b4a6ec88b914c80976de-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 192450b6e874b4a6ec88b914c80976de columnFamilyName c 2024-11-24T04:26:07,347 DEBUG [StoreOpener-192450b6e874b4a6ec88b914c80976de-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:26:07,347 INFO [StoreOpener-192450b6e874b4a6ec88b914c80976de-1 {}] regionserver.HStore(327): Store=192450b6e874b4a6ec88b914c80976de/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:26:07,348 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 192450b6e874b4a6ec88b914c80976de 2024-11-24T04:26:07,348 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44611/hbase/data/default/testReplayEditsWrittenViaHRegion/192450b6e874b4a6ec88b914c80976de 2024-11-24T04:26:07,349 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44611/hbase/data/default/testReplayEditsWrittenViaHRegion/192450b6e874b4a6ec88b914c80976de 2024-11-24T04:26:07,350 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 192450b6e874b4a6ec88b914c80976de 2024-11-24T04:26:07,350 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 192450b6e874b4a6ec88b914c80976de 2024-11-24T04:26:07,350 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-24T04:26:07,351 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 192450b6e874b4a6ec88b914c80976de 2024-11-24T04:26:07,353 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44611/hbase/data/default/testReplayEditsWrittenViaHRegion/192450b6e874b4a6ec88b914c80976de/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T04:26:07,354 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 192450b6e874b4a6ec88b914c80976de; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75015097, jitterRate=0.11781205236911774}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-24T04:26:07,354 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 192450b6e874b4a6ec88b914c80976de: Writing region info on filesystem at 1732422367341Initializing all the Stores at 1732422367342 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732422367342Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732422367342Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732422367342Cleaning up temporary data from old regions at 1732422367350 (+8 ms)Region opened successfully at 1732422367354 (+4 ms) 2024-11-24T04:26:07,354 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 192450b6e874b4a6ec88b914c80976de, disabling compactions & flushes 2024-11-24T04:26:07,354 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1732422367307.192450b6e874b4a6ec88b914c80976de. 2024-11-24T04:26:07,354 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1732422367307.192450b6e874b4a6ec88b914c80976de. 2024-11-24T04:26:07,354 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1732422367307.192450b6e874b4a6ec88b914c80976de. after waiting 0 ms 2024-11-24T04:26:07,354 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1732422367307.192450b6e874b4a6ec88b914c80976de. 2024-11-24T04:26:07,354 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1732422367307.192450b6e874b4a6ec88b914c80976de. 2024-11-24T04:26:07,355 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 192450b6e874b4a6ec88b914c80976de: Waiting for close lock at 1732422367354Disabling compacts and flushes for region at 1732422367354Disabling writes for close at 1732422367354Writing region close event to WAL at 1732422367354Closed at 1732422367354 2024-11-24T04:26:07,357 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34457 is added to blk_1073741925_1104 (size=95) 2024-11-24T04:26:07,357 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36069 is added to blk_1073741925_1104 (size=95) 2024-11-24T04:26:07,357 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44781 is added to blk_1073741925_1104 (size=95) 2024-11-24T04:26:07,359 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-24T04:26:07,359 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-09282742:(num 1732422367311) 2024-11-24T04:26:07,359 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-24T04:26:07,360 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:44611/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732422367306, archiveDir=hdfs://localhost:44611/hbase/oldWALs, maxLogs=32 2024-11-24T04:26:07,375 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732422367306/wal.1732422367361, exclude list is [], retry=0 2024-11-24T04:26:07,377 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34457,DS-20bb5930-23a3-441d-bdb2-015c0d56b2ec,DISK] 2024-11-24T04:26:07,377 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36069,DS-11c8f594-4fc2-47e4-b222-092ec50eb991,DISK] 2024-11-24T04:26:07,377 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44781,DS-54254ac8-0fae-43f7-ab06-53faa0e05dd7,DISK] 2024-11-24T04:26:07,378 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732422367306/wal.1732422367361 2024-11-24T04:26:07,379 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33347:33347),(127.0.0.1/127.0.0.1:41009:41009),(127.0.0.1/127.0.0.1:35691:35691)] 2024-11-24T04:26:07,379 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 192450b6e874b4a6ec88b914c80976de, NAME => 'testReplayEditsWrittenViaHRegion,,1732422367307.192450b6e874b4a6ec88b914c80976de.', STARTKEY => '', ENDKEY => ''} 2024-11-24T04:26:07,379 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1732422367307.192450b6e874b4a6ec88b914c80976de.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T04:26:07,379 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 192450b6e874b4a6ec88b914c80976de 2024-11-24T04:26:07,379 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 192450b6e874b4a6ec88b914c80976de 2024-11-24T04:26:07,380 INFO [StoreOpener-192450b6e874b4a6ec88b914c80976de-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 192450b6e874b4a6ec88b914c80976de 2024-11-24T04:26:07,381 INFO [StoreOpener-192450b6e874b4a6ec88b914c80976de-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 192450b6e874b4a6ec88b914c80976de columnFamilyName a 2024-11-24T04:26:07,381 DEBUG [StoreOpener-192450b6e874b4a6ec88b914c80976de-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:26:07,382 INFO [StoreOpener-192450b6e874b4a6ec88b914c80976de-1 {}] regionserver.HStore(327): Store=192450b6e874b4a6ec88b914c80976de/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:26:07,382 INFO [StoreOpener-192450b6e874b4a6ec88b914c80976de-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 192450b6e874b4a6ec88b914c80976de 2024-11-24T04:26:07,382 INFO [StoreOpener-192450b6e874b4a6ec88b914c80976de-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 192450b6e874b4a6ec88b914c80976de columnFamilyName b 2024-11-24T04:26:07,382 DEBUG [StoreOpener-192450b6e874b4a6ec88b914c80976de-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:26:07,383 INFO [StoreOpener-192450b6e874b4a6ec88b914c80976de-1 {}] regionserver.HStore(327): Store=192450b6e874b4a6ec88b914c80976de/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:26:07,383 INFO [StoreOpener-192450b6e874b4a6ec88b914c80976de-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 192450b6e874b4a6ec88b914c80976de 2024-11-24T04:26:07,383 INFO [StoreOpener-192450b6e874b4a6ec88b914c80976de-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 192450b6e874b4a6ec88b914c80976de columnFamilyName c 2024-11-24T04:26:07,383 DEBUG [StoreOpener-192450b6e874b4a6ec88b914c80976de-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:26:07,384 INFO [StoreOpener-192450b6e874b4a6ec88b914c80976de-1 {}] regionserver.HStore(327): Store=192450b6e874b4a6ec88b914c80976de/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:26:07,384 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 192450b6e874b4a6ec88b914c80976de 2024-11-24T04:26:07,385 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44611/hbase/data/default/testReplayEditsWrittenViaHRegion/192450b6e874b4a6ec88b914c80976de 2024-11-24T04:26:07,385 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44611/hbase/data/default/testReplayEditsWrittenViaHRegion/192450b6e874b4a6ec88b914c80976de 2024-11-24T04:26:07,386 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 192450b6e874b4a6ec88b914c80976de 2024-11-24T04:26:07,386 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 192450b6e874b4a6ec88b914c80976de 2024-11-24T04:26:07,387 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-24T04:26:07,388 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 192450b6e874b4a6ec88b914c80976de 2024-11-24T04:26:07,388 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 192450b6e874b4a6ec88b914c80976de; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69867759, jitterRate=0.04111073911190033}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-24T04:26:07,389 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 192450b6e874b4a6ec88b914c80976de: Writing region info on filesystem at 1732422367379Initializing all the Stores at 1732422367380 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732422367380Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732422367380Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732422367380Cleaning up temporary data from old regions at 1732422367386 (+6 ms)Region opened successfully at 1732422367389 (+3 ms) 2024-11-24T04:26:07,395 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 192450b6e874b4a6ec88b914c80976de 3/3 column families, dataSize=870 B heapSize=2.31 KB 2024-11-24T04:26:07,411 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44611/hbase/data/default/testReplayEditsWrittenViaHRegion/192450b6e874b4a6ec88b914c80976de/.tmp/a/080192b0d6b544a2b0be17b5237faddf is 91, key is testReplayEditsWrittenViaHRegion/a:x0/1732422367389/Put/seqid=0 2024-11-24T04:26:07,416 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44781 is added to blk_1073741928_1107 (size=5958) 2024-11-24T04:26:07,416 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34457 is added to blk_1073741928_1107 (size=5958) 2024-11-24T04:26:07,417 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36069 is added to blk_1073741928_1107 (size=5958) 2024-11-24T04:26:07,417 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:44611/hbase/data/default/testReplayEditsWrittenViaHRegion/192450b6e874b4a6ec88b914c80976de/.tmp/a/080192b0d6b544a2b0be17b5237faddf 2024-11-24T04:26:07,422 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44611/hbase/data/default/testReplayEditsWrittenViaHRegion/192450b6e874b4a6ec88b914c80976de/.tmp/a/080192b0d6b544a2b0be17b5237faddf as hdfs://localhost:44611/hbase/data/default/testReplayEditsWrittenViaHRegion/192450b6e874b4a6ec88b914c80976de/a/080192b0d6b544a2b0be17b5237faddf 2024-11-24T04:26:07,427 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44611/hbase/data/default/testReplayEditsWrittenViaHRegion/192450b6e874b4a6ec88b914c80976de/a/080192b0d6b544a2b0be17b5237faddf, entries=10, sequenceid=13, filesize=5.8 K 2024-11-24T04:26:07,428 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~870 B/870, heapSize ~1.80 KB/1840, currentSize=0 B/0 for 192450b6e874b4a6ec88b914c80976de in 33ms, sequenceid=13, compaction requested=false 2024-11-24T04:26:07,428 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 192450b6e874b4a6ec88b914c80976de: 2024-11-24T04:26:07,444 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 192450b6e874b4a6ec88b914c80976de, disabling compactions & flushes 2024-11-24T04:26:07,444 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1732422367307.192450b6e874b4a6ec88b914c80976de. 2024-11-24T04:26:07,444 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1732422367307.192450b6e874b4a6ec88b914c80976de. 2024-11-24T04:26:07,444 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1732422367307.192450b6e874b4a6ec88b914c80976de. after waiting 0 ms 2024-11-24T04:26:07,444 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1732422367307.192450b6e874b4a6ec88b914c80976de. 2024-11-24T04:26:07,445 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 1740 in region testReplayEditsWrittenViaHRegion,,1732422367307.192450b6e874b4a6ec88b914c80976de. 2024-11-24T04:26:07,445 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1732422367307.192450b6e874b4a6ec88b914c80976de. 2024-11-24T04:26:07,445 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 192450b6e874b4a6ec88b914c80976de: Waiting for close lock at 1732422367444Disabling compacts and flushes for region at 1732422367444Disabling writes for close at 1732422367444Writing region close event to WAL at 1732422367445 (+1 ms)Closed at 1732422367445 2024-11-24T04:26:07,448 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34457 is added to blk_1073741927_1106 (size=3347) 2024-11-24T04:26:07,449 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36069 is added to blk_1073741927_1106 (size=3347) 2024-11-24T04:26:07,449 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44781 is added to blk_1073741927_1106 (size=3347) 2024-11-24T04:26:07,471 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:44611/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732422367306/wal.1732422367361, size=3.3 K (3347bytes) 2024-11-24T04:26:07,471 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:44611/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732422367306/wal.1732422367361 2024-11-24T04:26:07,471 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:44611/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732422367306/wal.1732422367361 after 0ms 2024-11-24T04:26:07,474 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:44611/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732422367306/wal.1732422367361: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-24T04:26:07,474 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:44611/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732422367306/wal.1732422367361 took 4ms 2024-11-24T04:26:07,476 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:44611/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732422367306/wal.1732422367361 so closing down 2024-11-24T04:26:07,476 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-24T04:26:07,477 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1732422367361.temp 2024-11-24T04:26:07,478 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:44611/hbase/data/default/testReplayEditsWrittenViaHRegion/192450b6e874b4a6ec88b914c80976de/recovered.edits/0000000000000000003-wal.1732422367361.temp 2024-11-24T04:26:07,478 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-24T04:26:07,484 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36069 is added to blk_1073741929_1108 (size=2944) 2024-11-24T04:26:07,484 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44781 is added to blk_1073741929_1108 (size=2944) 2024-11-24T04:26:07,485 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34457 is added to blk_1073741929_1108 (size=2944) 2024-11-24T04:26:07,485 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:44611/hbase/data/default/testReplayEditsWrittenViaHRegion/192450b6e874b4a6ec88b914c80976de/recovered.edits/0000000000000000003-wal.1732422367361.temp (wrote 30 edits, skipped 0 edits in 0 ms) 2024-11-24T04:26:07,486 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:44611/hbase/data/default/testReplayEditsWrittenViaHRegion/192450b6e874b4a6ec88b914c80976de/recovered.edits/0000000000000000003-wal.1732422367361.temp to hdfs://localhost:44611/hbase/data/default/testReplayEditsWrittenViaHRegion/192450b6e874b4a6ec88b914c80976de/recovered.edits/0000000000000000035 2024-11-24T04:26:07,487 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 32 edits across 1 Regions in 13 ms; skipped=2; WAL=hdfs://localhost:44611/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732422367306/wal.1732422367361, size=3.3 K, length=3347, corrupted=false, cancelled=false 2024-11-24T04:26:07,487 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:44611/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732422367306/wal.1732422367361, journal: Splitting hdfs://localhost:44611/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732422367306/wal.1732422367361, size=3.3 K (3347bytes) at 1732422367471Finishing writing output for hdfs://localhost:44611/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732422367306/wal.1732422367361 so closing down at 1732422367476 (+5 ms)Creating recovered edits writer path=hdfs://localhost:44611/hbase/data/default/testReplayEditsWrittenViaHRegion/192450b6e874b4a6ec88b914c80976de/recovered.edits/0000000000000000003-wal.1732422367361.temp at 1732422367478 (+2 ms)3 split writer threads finished at 1732422367478Closed recovered edits writer path=hdfs://localhost:44611/hbase/data/default/testReplayEditsWrittenViaHRegion/192450b6e874b4a6ec88b914c80976de/recovered.edits/0000000000000000003-wal.1732422367361.temp (wrote 30 edits, skipped 0 edits in 0 ms) at 1732422367485 (+7 ms)Rename recovered edits hdfs://localhost:44611/hbase/data/default/testReplayEditsWrittenViaHRegion/192450b6e874b4a6ec88b914c80976de/recovered.edits/0000000000000000003-wal.1732422367361.temp to hdfs://localhost:44611/hbase/data/default/testReplayEditsWrittenViaHRegion/192450b6e874b4a6ec88b914c80976de/recovered.edits/0000000000000000035 at 1732422367486 (+1 ms)Processed 32 edits across 1 Regions in 13 ms; skipped=2; WAL=hdfs://localhost:44611/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732422367306/wal.1732422367361, size=3.3 K, length=3347, corrupted=false, cancelled=false at 1732422367487 (+1 ms) 2024-11-24T04:26:07,488 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:44611/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732422367306/wal.1732422367361 to hdfs://localhost:44611/hbase/oldWALs/wal.1732422367361 2024-11-24T04:26:07,489 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:44611/hbase/data/default/testReplayEditsWrittenViaHRegion/192450b6e874b4a6ec88b914c80976de/recovered.edits/0000000000000000035 2024-11-24T04:26:07,489 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-24T04:26:07,491 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:44611/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732422367306, archiveDir=hdfs://localhost:44611/hbase/oldWALs, maxLogs=32 2024-11-24T04:26:07,498 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-24T04:26:07,504 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732422367306/wal.1732422367491, exclude list is [], retry=0 2024-11-24T04:26:07,507 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36069,DS-11c8f594-4fc2-47e4-b222-092ec50eb991,DISK] 2024-11-24T04:26:07,507 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34457,DS-20bb5930-23a3-441d-bdb2-015c0d56b2ec,DISK] 2024-11-24T04:26:07,507 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44781,DS-54254ac8-0fae-43f7-ab06-53faa0e05dd7,DISK] 2024-11-24T04:26:07,509 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732422367306/wal.1732422367491 2024-11-24T04:26:07,510 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41009:41009),(127.0.0.1/127.0.0.1:33347:33347),(127.0.0.1/127.0.0.1:35691:35691)] 2024-11-24T04:26:07,510 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 192450b6e874b4a6ec88b914c80976de, NAME => 'testReplayEditsWrittenViaHRegion,,1732422367307.192450b6e874b4a6ec88b914c80976de.', STARTKEY => '', ENDKEY => ''} 2024-11-24T04:26:07,510 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1732422367307.192450b6e874b4a6ec88b914c80976de.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T04:26:07,511 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 192450b6e874b4a6ec88b914c80976de 2024-11-24T04:26:07,511 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 192450b6e874b4a6ec88b914c80976de 2024-11-24T04:26:07,512 INFO [StoreOpener-192450b6e874b4a6ec88b914c80976de-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 192450b6e874b4a6ec88b914c80976de 2024-11-24T04:26:07,513 INFO [StoreOpener-192450b6e874b4a6ec88b914c80976de-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 192450b6e874b4a6ec88b914c80976de columnFamilyName a 2024-11-24T04:26:07,513 DEBUG [StoreOpener-192450b6e874b4a6ec88b914c80976de-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:26:07,518 DEBUG [StoreOpener-192450b6e874b4a6ec88b914c80976de-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:44611/hbase/data/default/testReplayEditsWrittenViaHRegion/192450b6e874b4a6ec88b914c80976de/a/080192b0d6b544a2b0be17b5237faddf 2024-11-24T04:26:07,518 INFO [StoreOpener-192450b6e874b4a6ec88b914c80976de-1 {}] regionserver.HStore(327): Store=192450b6e874b4a6ec88b914c80976de/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:26:07,519 INFO [StoreOpener-192450b6e874b4a6ec88b914c80976de-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 192450b6e874b4a6ec88b914c80976de 2024-11-24T04:26:07,519 INFO [StoreOpener-192450b6e874b4a6ec88b914c80976de-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 192450b6e874b4a6ec88b914c80976de columnFamilyName b 2024-11-24T04:26:07,519 DEBUG [StoreOpener-192450b6e874b4a6ec88b914c80976de-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:26:07,520 INFO [StoreOpener-192450b6e874b4a6ec88b914c80976de-1 {}] regionserver.HStore(327): Store=192450b6e874b4a6ec88b914c80976de/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:26:07,520 INFO [StoreOpener-192450b6e874b4a6ec88b914c80976de-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 192450b6e874b4a6ec88b914c80976de 2024-11-24T04:26:07,521 INFO [StoreOpener-192450b6e874b4a6ec88b914c80976de-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 192450b6e874b4a6ec88b914c80976de columnFamilyName c 2024-11-24T04:26:07,521 DEBUG [StoreOpener-192450b6e874b4a6ec88b914c80976de-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:26:07,521 INFO [StoreOpener-192450b6e874b4a6ec88b914c80976de-1 {}] regionserver.HStore(327): Store=192450b6e874b4a6ec88b914c80976de/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:26:07,521 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 192450b6e874b4a6ec88b914c80976de 2024-11-24T04:26:07,522 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44611/hbase/data/default/testReplayEditsWrittenViaHRegion/192450b6e874b4a6ec88b914c80976de 2024-11-24T04:26:07,524 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:44611/hbase/data/default/testReplayEditsWrittenViaHRegion/192450b6e874b4a6ec88b914c80976de 2024-11-24T04:26:07,524 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:44611/hbase/data/default/testReplayEditsWrittenViaHRegion/192450b6e874b4a6ec88b914c80976de/recovered.edits/0000000000000000035 2024-11-24T04:26:07,526 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:44611/hbase/data/default/testReplayEditsWrittenViaHRegion/192450b6e874b4a6ec88b914c80976de/recovered.edits/0000000000000000035: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-24T04:26:07,528 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 20, skipped 10, firstSequenceIdInLog=3, maxSequenceIdInLog=35, path=hdfs://localhost:44611/hbase/data/default/testReplayEditsWrittenViaHRegion/192450b6e874b4a6ec88b914c80976de/recovered.edits/0000000000000000035 2024-11-24T04:26:07,528 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 192450b6e874b4a6ec88b914c80976de 3/3 column families, dataSize=1.70 KB heapSize=3.88 KB 2024-11-24T04:26:07,543 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44611/hbase/data/default/testReplayEditsWrittenViaHRegion/192450b6e874b4a6ec88b914c80976de/.tmp/b/0eeab214732d48a9be20733a9e05e7b8 is 91, key is testReplayEditsWrittenViaHRegion/b:x0/1732422367428/Put/seqid=0 2024-11-24T04:26:07,549 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34457 is added to blk_1073741931_1110 (size=5958) 2024-11-24T04:26:07,550 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36069 is added to blk_1073741931_1110 (size=5958) 2024-11-24T04:26:07,550 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44781 is added to blk_1073741931_1110 (size=5958) 2024-11-24T04:26:07,551 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=35 (bloomFilter=true), to=hdfs://localhost:44611/hbase/data/default/testReplayEditsWrittenViaHRegion/192450b6e874b4a6ec88b914c80976de/.tmp/b/0eeab214732d48a9be20733a9e05e7b8 2024-11-24T04:26:07,570 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44611/hbase/data/default/testReplayEditsWrittenViaHRegion/192450b6e874b4a6ec88b914c80976de/.tmp/c/d7ac61d5f9d348fb9ce6a409fecc944b is 91, key is testReplayEditsWrittenViaHRegion/c:x0/1732422367435/Put/seqid=0 2024-11-24T04:26:07,576 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44781 is added to blk_1073741932_1111 (size=5958) 2024-11-24T04:26:07,576 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34457 is added to blk_1073741932_1111 (size=5958) 2024-11-24T04:26:07,577 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36069 is added to blk_1073741932_1111 (size=5958) 2024-11-24T04:26:07,577 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=35 (bloomFilter=true), to=hdfs://localhost:44611/hbase/data/default/testReplayEditsWrittenViaHRegion/192450b6e874b4a6ec88b914c80976de/.tmp/c/d7ac61d5f9d348fb9ce6a409fecc944b 2024-11-24T04:26:07,582 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44611/hbase/data/default/testReplayEditsWrittenViaHRegion/192450b6e874b4a6ec88b914c80976de/.tmp/b/0eeab214732d48a9be20733a9e05e7b8 as hdfs://localhost:44611/hbase/data/default/testReplayEditsWrittenViaHRegion/192450b6e874b4a6ec88b914c80976de/b/0eeab214732d48a9be20733a9e05e7b8 2024-11-24T04:26:07,588 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44611/hbase/data/default/testReplayEditsWrittenViaHRegion/192450b6e874b4a6ec88b914c80976de/b/0eeab214732d48a9be20733a9e05e7b8, entries=10, sequenceid=35, filesize=5.8 K 2024-11-24T04:26:07,589 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44611/hbase/data/default/testReplayEditsWrittenViaHRegion/192450b6e874b4a6ec88b914c80976de/.tmp/c/d7ac61d5f9d348fb9ce6a409fecc944b as hdfs://localhost:44611/hbase/data/default/testReplayEditsWrittenViaHRegion/192450b6e874b4a6ec88b914c80976de/c/d7ac61d5f9d348fb9ce6a409fecc944b 2024-11-24T04:26:07,594 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44611/hbase/data/default/testReplayEditsWrittenViaHRegion/192450b6e874b4a6ec88b914c80976de/c/d7ac61d5f9d348fb9ce6a409fecc944b, entries=10, sequenceid=35, filesize=5.8 K 2024-11-24T04:26:07,594 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~1.70 KB/1740, heapSize ~3.59 KB/3680, currentSize=0 B/0 for 192450b6e874b4a6ec88b914c80976de in 66ms, sequenceid=35, compaction requested=false; wal=null 2024-11-24T04:26:07,595 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:44611/hbase/data/default/testReplayEditsWrittenViaHRegion/192450b6e874b4a6ec88b914c80976de/recovered.edits/0000000000000000035 2024-11-24T04:26:07,596 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 192450b6e874b4a6ec88b914c80976de 2024-11-24T04:26:07,596 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 192450b6e874b4a6ec88b914c80976de 2024-11-24T04:26:07,596 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-24T04:26:07,598 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 192450b6e874b4a6ec88b914c80976de 2024-11-24T04:26:07,599 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44611/hbase/data/default/testReplayEditsWrittenViaHRegion/192450b6e874b4a6ec88b914c80976de/recovered.edits/35.seqid, newMaxSeqId=35, maxSeqId=1 2024-11-24T04:26:07,600 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 192450b6e874b4a6ec88b914c80976de; next sequenceid=36; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68185412, jitterRate=0.016041815280914307}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-24T04:26:07,601 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 192450b6e874b4a6ec88b914c80976de: Writing region info on filesystem at 1732422367511Initializing all the Stores at 1732422367512 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732422367512Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732422367512Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732422367512Obtaining lock to block concurrent updates at 1732422367528 (+16 ms)Preparing flush snapshotting stores in 192450b6e874b4a6ec88b914c80976de at 1732422367528Finished memstore snapshotting testReplayEditsWrittenViaHRegion,,1732422367307.192450b6e874b4a6ec88b914c80976de., syncing WAL and waiting on mvcc, flushsize=dataSize=1740, getHeapSize=3920, getOffHeapSize=0, getCellsCount=20 at 1732422367528Flushing stores of testReplayEditsWrittenViaHRegion,,1732422367307.192450b6e874b4a6ec88b914c80976de. at 1732422367528Flushing 192450b6e874b4a6ec88b914c80976de/b: creating writer at 1732422367529 (+1 ms)Flushing 192450b6e874b4a6ec88b914c80976de/b: appending metadata at 1732422367543 (+14 ms)Flushing 192450b6e874b4a6ec88b914c80976de/b: closing flushed file at 1732422367543Flushing 192450b6e874b4a6ec88b914c80976de/c: creating writer at 1732422367555 (+12 ms)Flushing 192450b6e874b4a6ec88b914c80976de/c: appending metadata at 1732422367570 (+15 ms)Flushing 192450b6e874b4a6ec88b914c80976de/c: closing flushed file at 1732422367570Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6b08f3a2: reopening flushed file at 1732422367582 (+12 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4a67975b: reopening flushed file at 1732422367588 (+6 ms)Finished flush of dataSize ~1.70 KB/1740, heapSize ~3.59 KB/3680, currentSize=0 B/0 for 192450b6e874b4a6ec88b914c80976de in 66ms, sequenceid=35, compaction requested=false; wal=null at 1732422367594 (+6 ms)Cleaning up temporary data from old regions at 1732422367596 (+2 ms)Region opened successfully at 1732422367601 (+5 ms) 2024-11-24T04:26:07,667 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:44611/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732422367306/wal.1732422367491, size=0 (0bytes) 2024-11-24T04:26:07,667 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:44611/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732422367306/wal.1732422367491 might be still open, length is 0 2024-11-24T04:26:07,667 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:44611/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732422367306/wal.1732422367491 2024-11-24T04:26:07,667 WARN [IPC Server handler 2 on default port 44611 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732422367306/wal.1732422367491 has not been closed. Lease recovery is in progress. RecoveryId = 1112 for block blk_1073741930_1109 2024-11-24T04:26:07,667 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:44611/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732422367306/wal.1732422367491 after 0ms 2024-11-24T04:26:07,837 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1575425110_22 at /127.0.0.1:58722 [Receiving block BP-1091946017-172.17.0.2-1732422329789:blk_1073741930_1109] {}] datanode.DataXceiver(331): 127.0.0.1:36069:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:58722 dst: /127.0.0.1:36069 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:36069 remote=/127.0.0.1:58722]. Total timeout mills is 60000, 59790 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T04:26:07,838 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1575425110_22 at /127.0.0.1:55556 [Receiving block BP-1091946017-172.17.0.2-1732422329789:blk_1073741930_1109] {}] datanode.DataXceiver(331): 127.0.0.1:44781:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:55556 dst: /127.0.0.1:44781 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T04:26:07,838 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1575425110_22 at /127.0.0.1:46102 [Receiving block BP-1091946017-172.17.0.2-1732422329789:blk_1073741930_1109] {}] datanode.DataXceiver(331): 127.0.0.1:34457:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:46102 dst: /127.0.0.1:34457 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T04:26:07,840 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36069 is added to blk_1073741930_1112 (size=2936) 2024-11-24T04:26:07,841 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34457 is added to blk_1073741930_1112 (size=2936) 2024-11-24T04:26:11,668 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:44611/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732422367306/wal.1732422367491 after 4001ms 2024-11-24T04:26:11,671 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:44611/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732422367306/wal.1732422367491: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-24T04:26:11,672 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:44611/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732422367306/wal.1732422367491 took 4006ms 2024-11-24T04:26:11,674 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:44611/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732422367306/wal.1732422367491; continuing. 2024-11-24T04:26:11,674 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:44611/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732422367306/wal.1732422367491 so closing down 2024-11-24T04:26:11,674 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-24T04:26:11,675 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000037-wal.1732422367491.temp 2024-11-24T04:26:11,677 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:44611/hbase/data/default/testReplayEditsWrittenViaHRegion/192450b6e874b4a6ec88b914c80976de/recovered.edits/0000000000000000037-wal.1732422367491.temp 2024-11-24T04:26:11,677 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-24T04:26:11,686 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44781 is added to blk_1073741933_1113 (size=2944) 2024-11-24T04:26:11,686 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34457 is added to blk_1073741933_1113 (size=2944) 2024-11-24T04:26:11,686 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36069 is added to blk_1073741933_1113 (size=2944) 2024-11-24T04:26:11,687 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:44611/hbase/data/default/testReplayEditsWrittenViaHRegion/192450b6e874b4a6ec88b914c80976de/recovered.edits/0000000000000000037-wal.1732422367491.temp (wrote 30 edits, skipped 0 edits in 0 ms) 2024-11-24T04:26:11,688 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:44611/hbase/data/default/testReplayEditsWrittenViaHRegion/192450b6e874b4a6ec88b914c80976de/recovered.edits/0000000000000000037-wal.1732422367491.temp to hdfs://localhost:44611/hbase/data/default/testReplayEditsWrittenViaHRegion/192450b6e874b4a6ec88b914c80976de/recovered.edits/0000000000000000066 2024-11-24T04:26:11,689 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 30 edits across 1 Regions in 16 ms; skipped=0; WAL=hdfs://localhost:44611/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732422367306/wal.1732422367491, size=0, length=0, corrupted=false, cancelled=false 2024-11-24T04:26:11,689 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:44611/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732422367306/wal.1732422367491, journal: Splitting hdfs://localhost:44611/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732422367306/wal.1732422367491, size=0 (0bytes) at 1732422367667Finishing writing output for hdfs://localhost:44611/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732422367306/wal.1732422367491 so closing down at 1732422371674 (+4007 ms)Creating recovered edits writer path=hdfs://localhost:44611/hbase/data/default/testReplayEditsWrittenViaHRegion/192450b6e874b4a6ec88b914c80976de/recovered.edits/0000000000000000037-wal.1732422367491.temp at 1732422371677 (+3 ms)3 split writer threads finished at 1732422371677Closed recovered edits writer path=hdfs://localhost:44611/hbase/data/default/testReplayEditsWrittenViaHRegion/192450b6e874b4a6ec88b914c80976de/recovered.edits/0000000000000000037-wal.1732422367491.temp (wrote 30 edits, skipped 0 edits in 0 ms) at 1732422371687 (+10 ms)Rename recovered edits hdfs://localhost:44611/hbase/data/default/testReplayEditsWrittenViaHRegion/192450b6e874b4a6ec88b914c80976de/recovered.edits/0000000000000000037-wal.1732422367491.temp to hdfs://localhost:44611/hbase/data/default/testReplayEditsWrittenViaHRegion/192450b6e874b4a6ec88b914c80976de/recovered.edits/0000000000000000066 at 1732422371688 (+1 ms)Processed 30 edits across 1 Regions in 16 ms; skipped=0; WAL=hdfs://localhost:44611/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732422367306/wal.1732422367491, size=0, length=0, corrupted=false, cancelled=false at 1732422371689 (+1 ms) 2024-11-24T04:26:11,690 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:44611/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732422367306/wal.1732422367491 to hdfs://localhost:44611/hbase/oldWALs/wal.1732422367491 2024-11-24T04:26:11,691 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:44611/hbase/data/default/testReplayEditsWrittenViaHRegion/192450b6e874b4a6ec88b914c80976de/recovered.edits/0000000000000000066 2024-11-24T04:26:11,691 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-24T04:26:11,692 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:44611/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732422367306, archiveDir=hdfs://localhost:44611/hbase/oldWALs, maxLogs=32 2024-11-24T04:26:11,706 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732422367306/wal.1732422371693, exclude list is [], retry=0 2024-11-24T04:26:11,708 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44781,DS-54254ac8-0fae-43f7-ab06-53faa0e05dd7,DISK] 2024-11-24T04:26:11,709 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36069,DS-11c8f594-4fc2-47e4-b222-092ec50eb991,DISK] 2024-11-24T04:26:11,709 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34457,DS-20bb5930-23a3-441d-bdb2-015c0d56b2ec,DISK] 2024-11-24T04:26:11,710 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732422367306/wal.1732422371693 2024-11-24T04:26:11,711 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35691:35691),(127.0.0.1/127.0.0.1:41009:41009),(127.0.0.1/127.0.0.1:33347:33347)] 2024-11-24T04:26:11,711 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1732422367307.192450b6e874b4a6ec88b914c80976de.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T04:26:11,712 INFO [StoreOpener-192450b6e874b4a6ec88b914c80976de-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 192450b6e874b4a6ec88b914c80976de 2024-11-24T04:26:11,713 INFO [StoreOpener-192450b6e874b4a6ec88b914c80976de-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 192450b6e874b4a6ec88b914c80976de columnFamilyName a 2024-11-24T04:26:11,713 DEBUG [StoreOpener-192450b6e874b4a6ec88b914c80976de-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:26:11,718 DEBUG [StoreOpener-192450b6e874b4a6ec88b914c80976de-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:44611/hbase/data/default/testReplayEditsWrittenViaHRegion/192450b6e874b4a6ec88b914c80976de/a/080192b0d6b544a2b0be17b5237faddf 2024-11-24T04:26:11,718 INFO [StoreOpener-192450b6e874b4a6ec88b914c80976de-1 {}] regionserver.HStore(327): Store=192450b6e874b4a6ec88b914c80976de/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:26:11,718 INFO [StoreOpener-192450b6e874b4a6ec88b914c80976de-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 192450b6e874b4a6ec88b914c80976de 2024-11-24T04:26:11,719 INFO [StoreOpener-192450b6e874b4a6ec88b914c80976de-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 192450b6e874b4a6ec88b914c80976de columnFamilyName b 2024-11-24T04:26:11,719 DEBUG [StoreOpener-192450b6e874b4a6ec88b914c80976de-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:26:11,724 DEBUG [StoreOpener-192450b6e874b4a6ec88b914c80976de-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:44611/hbase/data/default/testReplayEditsWrittenViaHRegion/192450b6e874b4a6ec88b914c80976de/b/0eeab214732d48a9be20733a9e05e7b8 2024-11-24T04:26:11,724 INFO [StoreOpener-192450b6e874b4a6ec88b914c80976de-1 {}] regionserver.HStore(327): Store=192450b6e874b4a6ec88b914c80976de/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:26:11,724 INFO [StoreOpener-192450b6e874b4a6ec88b914c80976de-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 192450b6e874b4a6ec88b914c80976de 2024-11-24T04:26:11,725 INFO [StoreOpener-192450b6e874b4a6ec88b914c80976de-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 192450b6e874b4a6ec88b914c80976de columnFamilyName c 2024-11-24T04:26:11,725 DEBUG [StoreOpener-192450b6e874b4a6ec88b914c80976de-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:26:11,730 DEBUG [StoreOpener-192450b6e874b4a6ec88b914c80976de-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:44611/hbase/data/default/testReplayEditsWrittenViaHRegion/192450b6e874b4a6ec88b914c80976de/c/d7ac61d5f9d348fb9ce6a409fecc944b 2024-11-24T04:26:11,730 INFO [StoreOpener-192450b6e874b4a6ec88b914c80976de-1 {}] regionserver.HStore(327): Store=192450b6e874b4a6ec88b914c80976de/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:26:11,730 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 192450b6e874b4a6ec88b914c80976de 2024-11-24T04:26:11,731 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44611/hbase/data/default/testReplayEditsWrittenViaHRegion/192450b6e874b4a6ec88b914c80976de 2024-11-24T04:26:11,732 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:44611/hbase/data/default/testReplayEditsWrittenViaHRegion/192450b6e874b4a6ec88b914c80976de 2024-11-24T04:26:11,732 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:44611/hbase/data/default/testReplayEditsWrittenViaHRegion/192450b6e874b4a6ec88b914c80976de/recovered.edits/0000000000000000066 2024-11-24T04:26:11,734 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:44611/hbase/data/default/testReplayEditsWrittenViaHRegion/192450b6e874b4a6ec88b914c80976de/recovered.edits/0000000000000000066: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-24T04:26:11,739 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 30, skipped 0, firstSequenceIdInLog=37, maxSequenceIdInLog=66, path=hdfs://localhost:44611/hbase/data/default/testReplayEditsWrittenViaHRegion/192450b6e874b4a6ec88b914c80976de/recovered.edits/0000000000000000066 2024-11-24T04:26:11,739 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 192450b6e874b4a6ec88b914c80976de 3/3 column families, dataSize=2.55 KB heapSize=5.44 KB 2024-11-24T04:26:11,754 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44611/hbase/data/default/testReplayEditsWrittenViaHRegion/192450b6e874b4a6ec88b914c80976de/.tmp/a/ab5235d11918472e9e38d1e9b20eeaa9 is 91, key is testReplayEditsWrittenViaHRegion/a:y0/1732422367607/Put/seqid=0 2024-11-24T04:26:11,761 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36069 is added to blk_1073741935_1115 (size=5958) 2024-11-24T04:26:11,761 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44781 is added to blk_1073741935_1115 (size=5958) 2024-11-24T04:26:11,761 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34457 is added to blk_1073741935_1115 (size=5958) 2024-11-24T04:26:11,762 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:44611/hbase/data/default/testReplayEditsWrittenViaHRegion/192450b6e874b4a6ec88b914c80976de/.tmp/a/ab5235d11918472e9e38d1e9b20eeaa9 2024-11-24T04:26:11,780 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44611/hbase/data/default/testReplayEditsWrittenViaHRegion/192450b6e874b4a6ec88b914c80976de/.tmp/b/430e678d10b44ffcace442a428f0fc9c is 91, key is testReplayEditsWrittenViaHRegion/b:y0/1732422367614/Put/seqid=0 2024-11-24T04:26:11,786 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36069 is added to blk_1073741936_1116 (size=5958) 2024-11-24T04:26:11,786 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34457 is added to blk_1073741936_1116 (size=5958) 2024-11-24T04:26:11,786 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44781 is added to blk_1073741936_1116 (size=5958) 2024-11-24T04:26:11,786 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:44611/hbase/data/default/testReplayEditsWrittenViaHRegion/192450b6e874b4a6ec88b914c80976de/.tmp/b/430e678d10b44ffcace442a428f0fc9c 2024-11-24T04:26:11,806 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44611/hbase/data/default/testReplayEditsWrittenViaHRegion/192450b6e874b4a6ec88b914c80976de/.tmp/c/fabfe0c542e74009b70d6b991a68b7ff is 91, key is testReplayEditsWrittenViaHRegion/c:y0/1732422367621/Put/seqid=0 2024-11-24T04:26:11,811 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44781 is added to blk_1073741937_1117 (size=5958) 2024-11-24T04:26:11,812 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36069 is added to blk_1073741937_1117 (size=5958) 2024-11-24T04:26:11,812 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34457 is added to blk_1073741937_1117 (size=5958) 2024-11-24T04:26:11,813 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:44611/hbase/data/default/testReplayEditsWrittenViaHRegion/192450b6e874b4a6ec88b914c80976de/.tmp/c/fabfe0c542e74009b70d6b991a68b7ff 2024-11-24T04:26:11,818 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44611/hbase/data/default/testReplayEditsWrittenViaHRegion/192450b6e874b4a6ec88b914c80976de/.tmp/a/ab5235d11918472e9e38d1e9b20eeaa9 as hdfs://localhost:44611/hbase/data/default/testReplayEditsWrittenViaHRegion/192450b6e874b4a6ec88b914c80976de/a/ab5235d11918472e9e38d1e9b20eeaa9 2024-11-24T04:26:11,823 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44611/hbase/data/default/testReplayEditsWrittenViaHRegion/192450b6e874b4a6ec88b914c80976de/a/ab5235d11918472e9e38d1e9b20eeaa9, entries=10, sequenceid=66, filesize=5.8 K 2024-11-24T04:26:11,824 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44611/hbase/data/default/testReplayEditsWrittenViaHRegion/192450b6e874b4a6ec88b914c80976de/.tmp/b/430e678d10b44ffcace442a428f0fc9c as hdfs://localhost:44611/hbase/data/default/testReplayEditsWrittenViaHRegion/192450b6e874b4a6ec88b914c80976de/b/430e678d10b44ffcace442a428f0fc9c 2024-11-24T04:26:11,829 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44611/hbase/data/default/testReplayEditsWrittenViaHRegion/192450b6e874b4a6ec88b914c80976de/b/430e678d10b44ffcace442a428f0fc9c, entries=10, sequenceid=66, filesize=5.8 K 2024-11-24T04:26:11,830 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44611/hbase/data/default/testReplayEditsWrittenViaHRegion/192450b6e874b4a6ec88b914c80976de/.tmp/c/fabfe0c542e74009b70d6b991a68b7ff as hdfs://localhost:44611/hbase/data/default/testReplayEditsWrittenViaHRegion/192450b6e874b4a6ec88b914c80976de/c/fabfe0c542e74009b70d6b991a68b7ff 2024-11-24T04:26:11,834 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44611/hbase/data/default/testReplayEditsWrittenViaHRegion/192450b6e874b4a6ec88b914c80976de/c/fabfe0c542e74009b70d6b991a68b7ff, entries=10, sequenceid=66, filesize=5.8 K 2024-11-24T04:26:11,835 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~2.55 KB/2610, heapSize ~5.39 KB/5520, currentSize=0 B/0 for 192450b6e874b4a6ec88b914c80976de in 96ms, sequenceid=66, compaction requested=false; wal=null 2024-11-24T04:26:11,835 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:44611/hbase/data/default/testReplayEditsWrittenViaHRegion/192450b6e874b4a6ec88b914c80976de/recovered.edits/0000000000000000066 2024-11-24T04:26:11,836 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 192450b6e874b4a6ec88b914c80976de 2024-11-24T04:26:11,836 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 192450b6e874b4a6ec88b914c80976de 2024-11-24T04:26:11,837 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-24T04:26:11,838 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 192450b6e874b4a6ec88b914c80976de 2024-11-24T04:26:11,840 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44611/hbase/data/default/testReplayEditsWrittenViaHRegion/192450b6e874b4a6ec88b914c80976de/recovered.edits/66.seqid, newMaxSeqId=66, maxSeqId=35 2024-11-24T04:26:11,841 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 192450b6e874b4a6ec88b914c80976de; next sequenceid=67; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69983878, jitterRate=0.042841047048568726}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-24T04:26:11,841 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 192450b6e874b4a6ec88b914c80976de: Writing region info on filesystem at 1732422371711Initializing all the Stores at 1732422371712 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732422371712Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732422371712Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732422371712Obtaining lock to block concurrent updates at 1732422371739 (+27 ms)Preparing flush snapshotting stores in 192450b6e874b4a6ec88b914c80976de at 1732422371739Finished memstore snapshotting testReplayEditsWrittenViaHRegion,,1732422367307.192450b6e874b4a6ec88b914c80976de., syncing WAL and waiting on mvcc, flushsize=dataSize=2610, getHeapSize=5520, getOffHeapSize=0, getCellsCount=30 at 1732422371739Flushing stores of testReplayEditsWrittenViaHRegion,,1732422367307.192450b6e874b4a6ec88b914c80976de. at 1732422371739Flushing 192450b6e874b4a6ec88b914c80976de/a: creating writer at 1732422371739Flushing 192450b6e874b4a6ec88b914c80976de/a: appending metadata at 1732422371753 (+14 ms)Flushing 192450b6e874b4a6ec88b914c80976de/a: closing flushed file at 1732422371753Flushing 192450b6e874b4a6ec88b914c80976de/b: creating writer at 1732422371766 (+13 ms)Flushing 192450b6e874b4a6ec88b914c80976de/b: appending metadata at 1732422371780 (+14 ms)Flushing 192450b6e874b4a6ec88b914c80976de/b: closing flushed file at 1732422371780Flushing 192450b6e874b4a6ec88b914c80976de/c: creating writer at 1732422371791 (+11 ms)Flushing 192450b6e874b4a6ec88b914c80976de/c: appending metadata at 1732422371805 (+14 ms)Flushing 192450b6e874b4a6ec88b914c80976de/c: closing flushed file at 1732422371805Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@28af48ec: reopening flushed file at 1732422371817 (+12 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3df39927: reopening flushed file at 1732422371823 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@71652bb3: reopening flushed file at 1732422371829 (+6 ms)Finished flush of dataSize ~2.55 KB/2610, heapSize ~5.39 KB/5520, currentSize=0 B/0 for 192450b6e874b4a6ec88b914c80976de in 96ms, sequenceid=66, compaction requested=false; wal=null at 1732422371835 (+6 ms)Cleaning up temporary data from old regions at 1732422371836 (+1 ms)Region opened successfully at 1732422371841 (+5 ms) 2024-11-24T04:26:11,854 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 192450b6e874b4a6ec88b914c80976de, disabling compactions & flushes 2024-11-24T04:26:11,854 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1732422367307.192450b6e874b4a6ec88b914c80976de. 2024-11-24T04:26:11,854 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1732422367307.192450b6e874b4a6ec88b914c80976de. 2024-11-24T04:26:11,854 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1732422367307.192450b6e874b4a6ec88b914c80976de. after waiting 0 ms 2024-11-24T04:26:11,854 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1732422367307.192450b6e874b4a6ec88b914c80976de. 2024-11-24T04:26:11,856 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1732422367307.192450b6e874b4a6ec88b914c80976de. 2024-11-24T04:26:11,856 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 192450b6e874b4a6ec88b914c80976de: Waiting for close lock at 1732422371854Disabling compacts and flushes for region at 1732422371854Disabling writes for close at 1732422371854Writing region close event to WAL at 1732422371856 (+2 ms)Closed at 1732422371856 2024-11-24T04:26:11,859 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34457 is added to blk_1073741934_1114 (size=95) 2024-11-24T04:26:11,859 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36069 is added to blk_1073741934_1114 (size=95) 2024-11-24T04:26:11,859 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44781 is added to blk_1073741934_1114 (size=95) 2024-11-24T04:26:11,862 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-24T04:26:11,862 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1732422371693) 2024-11-24T04:26:11,878 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayValueCompression#testReplayEditsWrittenViaHRegion Thread=447 (was 442) Potentially hanging thread: LeaseRenewer:jenkinstestReplayEditsWrittenViaHRegion@localhost:44611 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-28-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1319272049) connection to localhost/127.0.0.1:44611 from jenkinstestReplayEditsWrittenViaHRegion java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-167666862_22 at /127.0.0.1:37520 [Waiting for operation #21] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-28-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-167666862_22 at /127.0.0.1:57450 [Waiting for operation #6] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-28-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-167666862_22 at /127.0.0.1:37580 [Waiting for operation #16] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1414 (was 1346) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=148 (was 161), ProcessCount=11 (was 11), AvailableMemoryMB=6055 (was 6085) 2024-11-24T04:26:11,878 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1414 is superior to 1024 2024-11-24T04:26:11,878 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-24T04:26:11,878 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-24T04:26:11,878 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-24T04:26:11,878 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T04:26:11,879 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T04:26:11,879 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T04:26:11,879 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-24T04:26:11,879 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1506073043, stopped=false 2024-11-24T04:26:11,880 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=fadb2fe0ba32,33969,1732422332893 2024-11-24T04:26:11,881 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35977-0x1014838d9450001, quorum=127.0.0.1:62017, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-24T04:26:11,881 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33969-0x1014838d9450000, quorum=127.0.0.1:62017, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-24T04:26:11,881 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45209-0x1014838d9450003, quorum=127.0.0.1:62017, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-24T04:26:11,882 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33969-0x1014838d9450000, quorum=127.0.0.1:62017, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T04:26:11,882 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35977-0x1014838d9450001, quorum=127.0.0.1:62017, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T04:26:11,882 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45209-0x1014838d9450003, quorum=127.0.0.1:62017, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T04:26:11,882 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-24T04:26:11,882 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-24T04:26:11,882 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-24T04:26:11,882 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T04:26:11,883 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'fadb2fe0ba32,35977,1732422333601' ***** 2024-11-24T04:26:11,883 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-24T04:26:11,883 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'fadb2fe0ba32,45209,1732422333751' ***** 2024-11-24T04:26:11,883 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-24T04:26:11,883 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:35977-0x1014838d9450001, quorum=127.0.0.1:62017, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-24T04:26:11,883 INFO [RS:2;fadb2fe0ba32:45209 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-24T04:26:11,883 INFO [RS:2;fadb2fe0ba32:45209 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-24T04:26:11,883 INFO [RS:2;fadb2fe0ba32:45209 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-24T04:26:11,883 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-24T04:26:11,883 INFO [RS:2;fadb2fe0ba32:45209 {}] regionserver.HRegionServer(3091): Received CLOSE for ceffd2386e7c72b432c9dfad6355a885 2024-11-24T04:26:11,883 INFO [RS:0;fadb2fe0ba32:35977 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-24T04:26:11,884 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-24T04:26:11,884 INFO [RS:0;fadb2fe0ba32:35977 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-24T04:26:11,884 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:33969-0x1014838d9450000, quorum=127.0.0.1:62017, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-24T04:26:11,884 INFO [RS:0;fadb2fe0ba32:35977 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-24T04:26:11,884 INFO [RS:0;fadb2fe0ba32:35977 {}] regionserver.HRegionServer(959): stopping server fadb2fe0ba32,35977,1732422333601 2024-11-24T04:26:11,884 INFO [RS:0;fadb2fe0ba32:35977 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-24T04:26:11,884 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:45209-0x1014838d9450003, quorum=127.0.0.1:62017, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-24T04:26:11,884 INFO [RS:0;fadb2fe0ba32:35977 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;fadb2fe0ba32:35977. 2024-11-24T04:26:11,884 INFO [RS:2;fadb2fe0ba32:45209 {}] regionserver.HRegionServer(959): stopping server fadb2fe0ba32,45209,1732422333751 2024-11-24T04:26:11,884 DEBUG [RS:0;fadb2fe0ba32:35977 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-24T04:26:11,884 INFO [RS:2;fadb2fe0ba32:45209 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-24T04:26:11,884 DEBUG [RS:0;fadb2fe0ba32:35977 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T04:26:11,884 INFO [RS:2;fadb2fe0ba32:45209 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:2;fadb2fe0ba32:45209. 2024-11-24T04:26:11,884 INFO [RS:0;fadb2fe0ba32:35977 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-24T04:26:11,884 DEBUG [RS:2;fadb2fe0ba32:45209 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-24T04:26:11,884 INFO [RS:0;fadb2fe0ba32:35977 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-24T04:26:11,884 DEBUG [RS:2;fadb2fe0ba32:45209 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T04:26:11,884 INFO [RS:0;fadb2fe0ba32:35977 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-24T04:26:11,884 INFO [RS:2;fadb2fe0ba32:45209 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-24T04:26:11,884 INFO [RS:0;fadb2fe0ba32:35977 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-24T04:26:11,885 DEBUG [RS:2;fadb2fe0ba32:45209 {}] regionserver.HRegionServer(1325): Online Regions={ceffd2386e7c72b432c9dfad6355a885=testReplayEditsAfterRegionMovedWithMultiCF,,1732422348811.ceffd2386e7c72b432c9dfad6355a885.} 2024-11-24T04:26:11,885 DEBUG [RS:2;fadb2fe0ba32:45209 {}] regionserver.HRegionServer(1351): Waiting on ceffd2386e7c72b432c9dfad6355a885 2024-11-24T04:26:11,885 INFO [RS:0;fadb2fe0ba32:35977 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-24T04:26:11,885 DEBUG [RS:0;fadb2fe0ba32:35977 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-24T04:26:11,885 DEBUG [RS_CLOSE_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing ceffd2386e7c72b432c9dfad6355a885, disabling compactions & flushes 2024-11-24T04:26:11,885 DEBUG [RS:0;fadb2fe0ba32:35977 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-24T04:26:11,885 DEBUG [RS_CLOSE_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-24T04:26:11,885 INFO [RS_CLOSE_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1732422348811.ceffd2386e7c72b432c9dfad6355a885. 2024-11-24T04:26:11,885 INFO [RS_CLOSE_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-24T04:26:11,885 DEBUG [RS_CLOSE_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732422348811.ceffd2386e7c72b432c9dfad6355a885. 2024-11-24T04:26:11,885 DEBUG [RS_CLOSE_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-24T04:26:11,885 DEBUG [RS_CLOSE_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-24T04:26:11,885 DEBUG [RS_CLOSE_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732422348811.ceffd2386e7c72b432c9dfad6355a885. after waiting 0 ms 2024-11-24T04:26:11,885 DEBUG [RS_CLOSE_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-24T04:26:11,885 DEBUG [RS_CLOSE_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1732422348811.ceffd2386e7c72b432c9dfad6355a885. 2024-11-24T04:26:11,885 INFO [RS_CLOSE_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=6.86 KB heapSize=11.45 KB 2024-11-24T04:26:11,887 INFO [regionserver/fadb2fe0ba32:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-24T04:26:11,890 DEBUG [RS_CLOSE_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44611/user/jenkins/test-data/33c38499-d2fe-b6f0-8d8b-5809759afd19/data/default/testReplayEditsAfterRegionMovedWithMultiCF/ceffd2386e7c72b432c9dfad6355a885/recovered.edits/20.seqid, newMaxSeqId=20, maxSeqId=17 2024-11-24T04:26:11,890 INFO [RS_CLOSE_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732422348811.ceffd2386e7c72b432c9dfad6355a885. 2024-11-24T04:26:11,890 DEBUG [RS_CLOSE_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for ceffd2386e7c72b432c9dfad6355a885: Waiting for close lock at 1732422371885Running coprocessor pre-close hooks at 1732422371885Disabling compacts and flushes for region at 1732422371885Disabling writes for close at 1732422371885Writing region close event to WAL at 1732422371886 (+1 ms)Running coprocessor post-close hooks at 1732422371890 (+4 ms)Closed at 1732422371890 2024-11-24T04:26:11,891 DEBUG [RS_CLOSE_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732422348811.ceffd2386e7c72b432c9dfad6355a885. 2024-11-24T04:26:11,914 DEBUG [RS_CLOSE_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44611/user/jenkins/test-data/33c38499-d2fe-b6f0-8d8b-5809759afd19/data/hbase/meta/1588230740/.tmp/info/15021cbb040144ca8b577a17d97a97cc is 205, key is testReplayEditsAfterRegionMovedWithMultiCF,,1732422348811.ceffd2386e7c72b432c9dfad6355a885./info:regioninfo/1732422351854/Put/seqid=0 2024-11-24T04:26:11,920 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44781 is added to blk_1073741938_1118 (size=8243) 2024-11-24T04:26:11,920 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36069 is added to blk_1073741938_1118 (size=8243) 2024-11-24T04:26:11,920 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34457 is added to blk_1073741938_1118 (size=8243) 2024-11-24T04:26:11,921 INFO [RS_CLOSE_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.65 KB at sequenceid=21 (bloomFilter=true), to=hdfs://localhost:44611/user/jenkins/test-data/33c38499-d2fe-b6f0-8d8b-5809759afd19/data/hbase/meta/1588230740/.tmp/info/15021cbb040144ca8b577a17d97a97cc 2024-11-24T04:26:11,941 DEBUG [RS_CLOSE_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44611/user/jenkins/test-data/33c38499-d2fe-b6f0-8d8b-5809759afd19/data/hbase/meta/1588230740/.tmp/ns/e07f781027344d8d96a03055a8afe213 is 43, key is default/ns:d/1732422335722/Put/seqid=0 2024-11-24T04:26:11,947 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34457 is added to blk_1073741939_1119 (size=5153) 2024-11-24T04:26:11,947 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36069 is added to blk_1073741939_1119 (size=5153) 2024-11-24T04:26:11,948 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44781 is added to blk_1073741939_1119 (size=5153) 2024-11-24T04:26:11,948 INFO [RS_CLOSE_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=21 (bloomFilter=true), to=hdfs://localhost:44611/user/jenkins/test-data/33c38499-d2fe-b6f0-8d8b-5809759afd19/data/hbase/meta/1588230740/.tmp/ns/e07f781027344d8d96a03055a8afe213 2024-11-24T04:26:11,967 DEBUG [RS_CLOSE_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44611/user/jenkins/test-data/33c38499-d2fe-b6f0-8d8b-5809759afd19/data/hbase/meta/1588230740/.tmp/table/ab26f2b184ab4c95b81e8d9eda9188f8 is 78, key is testReplayEditsAfterRegionMovedWithMultiCF/table:state/1732422349225/Put/seqid=0 2024-11-24T04:26:11,973 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34457 is added to blk_1073741940_1120 (size=5431) 2024-11-24T04:26:11,973 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44781 is added to blk_1073741940_1120 (size=5431) 2024-11-24T04:26:11,974 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36069 is added to blk_1073741940_1120 (size=5431) 2024-11-24T04:26:11,974 INFO [RS_CLOSE_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=148 B at sequenceid=21 (bloomFilter=true), to=hdfs://localhost:44611/user/jenkins/test-data/33c38499-d2fe-b6f0-8d8b-5809759afd19/data/hbase/meta/1588230740/.tmp/table/ab26f2b184ab4c95b81e8d9eda9188f8 2024-11-24T04:26:11,979 DEBUG [RS_CLOSE_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44611/user/jenkins/test-data/33c38499-d2fe-b6f0-8d8b-5809759afd19/data/hbase/meta/1588230740/.tmp/info/15021cbb040144ca8b577a17d97a97cc as hdfs://localhost:44611/user/jenkins/test-data/33c38499-d2fe-b6f0-8d8b-5809759afd19/data/hbase/meta/1588230740/info/15021cbb040144ca8b577a17d97a97cc 2024-11-24T04:26:11,981 INFO [regionserver/fadb2fe0ba32:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-24T04:26:11,984 INFO [RS_CLOSE_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44611/user/jenkins/test-data/33c38499-d2fe-b6f0-8d8b-5809759afd19/data/hbase/meta/1588230740/info/15021cbb040144ca8b577a17d97a97cc, entries=18, sequenceid=21, filesize=8.0 K 2024-11-24T04:26:11,985 DEBUG [RS_CLOSE_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44611/user/jenkins/test-data/33c38499-d2fe-b6f0-8d8b-5809759afd19/data/hbase/meta/1588230740/.tmp/ns/e07f781027344d8d96a03055a8afe213 as hdfs://localhost:44611/user/jenkins/test-data/33c38499-d2fe-b6f0-8d8b-5809759afd19/data/hbase/meta/1588230740/ns/e07f781027344d8d96a03055a8afe213 2024-11-24T04:26:11,990 INFO [RS_CLOSE_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44611/user/jenkins/test-data/33c38499-d2fe-b6f0-8d8b-5809759afd19/data/hbase/meta/1588230740/ns/e07f781027344d8d96a03055a8afe213, entries=2, sequenceid=21, filesize=5.0 K 2024-11-24T04:26:11,991 DEBUG [RS_CLOSE_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44611/user/jenkins/test-data/33c38499-d2fe-b6f0-8d8b-5809759afd19/data/hbase/meta/1588230740/.tmp/table/ab26f2b184ab4c95b81e8d9eda9188f8 as hdfs://localhost:44611/user/jenkins/test-data/33c38499-d2fe-b6f0-8d8b-5809759afd19/data/hbase/meta/1588230740/table/ab26f2b184ab4c95b81e8d9eda9188f8 2024-11-24T04:26:11,995 INFO [RS_CLOSE_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44611/user/jenkins/test-data/33c38499-d2fe-b6f0-8d8b-5809759afd19/data/hbase/meta/1588230740/table/ab26f2b184ab4c95b81e8d9eda9188f8, entries=2, sequenceid=21, filesize=5.3 K 2024-11-24T04:26:11,996 INFO [RS_CLOSE_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~6.86 KB/7029, heapSize ~11.16 KB/11424, currentSize=0 B/0 for 1588230740 in 111ms, sequenceid=21, compaction requested=false 2024-11-24T04:26:12,001 DEBUG [RS_CLOSE_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44611/user/jenkins/test-data/33c38499-d2fe-b6f0-8d8b-5809759afd19/data/hbase/meta/1588230740/recovered.edits/24.seqid, newMaxSeqId=24, maxSeqId=1 2024-11-24T04:26:12,001 DEBUG [RS_CLOSE_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-24T04:26:12,001 INFO [RS_CLOSE_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-24T04:26:12,001 DEBUG [RS_CLOSE_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732422371885Running coprocessor pre-close hooks at 1732422371885Disabling compacts and flushes for region at 1732422371885Disabling writes for close at 1732422371885Obtaining lock to block concurrent updates at 1732422371885Preparing flush snapshotting stores in 1588230740 at 1732422371885Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=7029, getHeapSize=11664, getOffHeapSize=0, getCellsCount=48 at 1732422371886 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1732422371886Flushing 1588230740/info: creating writer at 1732422371887 (+1 ms)Flushing 1588230740/info: appending metadata at 1732422371913 (+26 ms)Flushing 1588230740/info: closing flushed file at 1732422371913Flushing 1588230740/ns: creating writer at 1732422371926 (+13 ms)Flushing 1588230740/ns: appending metadata at 1732422371940 (+14 ms)Flushing 1588230740/ns: closing flushed file at 1732422371940Flushing 1588230740/table: creating writer at 1732422371953 (+13 ms)Flushing 1588230740/table: appending metadata at 1732422371967 (+14 ms)Flushing 1588230740/table: closing flushed file at 1732422371967Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@675181a2: reopening flushed file at 1732422371979 (+12 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2d640acf: reopening flushed file at 1732422371985 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@176c9a81: reopening flushed file at 1732422371990 (+5 ms)Finished flush of dataSize ~6.86 KB/7029, heapSize ~11.16 KB/11424, currentSize=0 B/0 for 1588230740 in 111ms, sequenceid=21, compaction requested=false at 1732422371996 (+6 ms)Writing region close event to WAL at 1732422371997 (+1 ms)Running coprocessor post-close hooks at 1732422372001 (+4 ms)Closed at 1732422372001 2024-11-24T04:26:12,002 DEBUG [RS_CLOSE_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-24T04:26:12,085 INFO [RS:2;fadb2fe0ba32:45209 {}] regionserver.HRegionServer(976): stopping server fadb2fe0ba32,45209,1732422333751; all regions closed. 2024-11-24T04:26:12,085 INFO [RS:0;fadb2fe0ba32:35977 {}] regionserver.HRegionServer(976): stopping server fadb2fe0ba32,35977,1732422333601; all regions closed. 2024-11-24T04:26:12,088 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36069 is added to blk_1073741833_1009 (size=721) 2024-11-24T04:26:12,088 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44781 is added to blk_1073741833_1009 (size=721) 2024-11-24T04:26:12,088 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34457 is added to blk_1073741833_1009 (size=721) 2024-11-24T04:26:12,089 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44781 is added to blk_1073741836_1012 (size=4719) 2024-11-24T04:26:12,089 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36069 is added to blk_1073741836_1012 (size=4719) 2024-11-24T04:26:12,089 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34457 is added to blk_1073741836_1012 (size=4719) 2024-11-24T04:26:12,091 DEBUG [RS:0;fadb2fe0ba32:35977 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/33c38499-d2fe-b6f0-8d8b-5809759afd19/oldWALs 2024-11-24T04:26:12,091 INFO [RS:0;fadb2fe0ba32:35977 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL fadb2fe0ba32%2C35977%2C1732422333601.meta:.meta(num 1732422335569) 2024-11-24T04:26:12,091 DEBUG [RS:2;fadb2fe0ba32:45209 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/33c38499-d2fe-b6f0-8d8b-5809759afd19/oldWALs 2024-11-24T04:26:12,091 INFO [RS:2;fadb2fe0ba32:45209 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL fadb2fe0ba32%2C45209%2C1732422333751:(num 1732422335218) 2024-11-24T04:26:12,091 DEBUG [RS:2;fadb2fe0ba32:45209 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T04:26:12,091 INFO [RS:2;fadb2fe0ba32:45209 {}] regionserver.LeaseManager(133): Closed leases 2024-11-24T04:26:12,092 INFO [RS:2;fadb2fe0ba32:45209 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-24T04:26:12,092 INFO [RS:2;fadb2fe0ba32:45209 {}] hbase.ChoreService(370): Chore service for: regionserver/fadb2fe0ba32:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-24T04:26:12,092 INFO [RS:2;fadb2fe0ba32:45209 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-24T04:26:12,092 INFO [regionserver/fadb2fe0ba32:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-24T04:26:12,092 INFO [RS:2;fadb2fe0ba32:45209 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-24T04:26:12,092 INFO [RS:2;fadb2fe0ba32:45209 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-24T04:26:12,092 INFO [RS:2;fadb2fe0ba32:45209 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-24T04:26:12,092 INFO [RS:2;fadb2fe0ba32:45209 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:45209 2024-11-24T04:26:12,094 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44781 is added to blk_1073741834_1010 (size=1704) 2024-11-24T04:26:12,094 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/33c38499-d2fe-b6f0-8d8b-5809759afd19/WALs/fadb2fe0ba32,35977,1732422333601/fadb2fe0ba32%2C35977%2C1732422333601.1732422335220 not finished, retry = 0 2024-11-24T04:26:12,094 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36069 is added to blk_1073741834_1010 (size=1704) 2024-11-24T04:26:12,095 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33969-0x1014838d9450000, quorum=127.0.0.1:62017, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-24T04:26:12,095 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45209-0x1014838d9450003, quorum=127.0.0.1:62017, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/fadb2fe0ba32,45209,1732422333751 2024-11-24T04:26:12,095 INFO [RS:2;fadb2fe0ba32:45209 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-24T04:26:12,095 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [fadb2fe0ba32,45209,1732422333751] 2024-11-24T04:26:12,097 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34457 is added to blk_1073741834_1010 (size=1704) 2024-11-24T04:26:12,101 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/fadb2fe0ba32,45209,1732422333751 already deleted, retry=false 2024-11-24T04:26:12,101 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; fadb2fe0ba32,45209,1732422333751 expired; onlineServers=1 2024-11-24T04:26:12,128 INFO [regionserver/fadb2fe0ba32:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-24T04:26:12,129 INFO [regionserver/fadb2fe0ba32:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-24T04:26:12,196 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45209-0x1014838d9450003, quorum=127.0.0.1:62017, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-24T04:26:12,196 INFO [RS:2;fadb2fe0ba32:45209 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-24T04:26:12,196 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45209-0x1014838d9450003, quorum=127.0.0.1:62017, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-24T04:26:12,196 INFO [RS:2;fadb2fe0ba32:45209 {}] regionserver.HRegionServer(1031): Exiting; stopping=fadb2fe0ba32,45209,1732422333751; zookeeper connection closed. 2024-11-24T04:26:12,197 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@4ba666cf {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@4ba666cf 2024-11-24T04:26:12,197 DEBUG [RS:0;fadb2fe0ba32:35977 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/33c38499-d2fe-b6f0-8d8b-5809759afd19/oldWALs 2024-11-24T04:26:12,197 INFO [RS:0;fadb2fe0ba32:35977 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL fadb2fe0ba32%2C35977%2C1732422333601:(num 1732422335220) 2024-11-24T04:26:12,197 DEBUG [RS:0;fadb2fe0ba32:35977 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T04:26:12,197 INFO [RS:0;fadb2fe0ba32:35977 {}] regionserver.LeaseManager(133): Closed leases 2024-11-24T04:26:12,197 INFO [RS:0;fadb2fe0ba32:35977 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-24T04:26:12,197 INFO [RS:0;fadb2fe0ba32:35977 {}] hbase.ChoreService(370): Chore service for: regionserver/fadb2fe0ba32:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-24T04:26:12,198 INFO [RS:0;fadb2fe0ba32:35977 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-24T04:26:12,198 INFO [regionserver/fadb2fe0ba32:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-24T04:26:12,198 INFO [RS:0;fadb2fe0ba32:35977 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:35977 2024-11-24T04:26:12,200 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35977-0x1014838d9450001, quorum=127.0.0.1:62017, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/fadb2fe0ba32,35977,1732422333601 2024-11-24T04:26:12,200 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33969-0x1014838d9450000, quorum=127.0.0.1:62017, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-24T04:26:12,200 INFO [RS:0;fadb2fe0ba32:35977 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-24T04:26:12,201 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [fadb2fe0ba32,35977,1732422333601] 2024-11-24T04:26:12,202 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/fadb2fe0ba32,35977,1732422333601 already deleted, retry=false 2024-11-24T04:26:12,202 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; fadb2fe0ba32,35977,1732422333601 expired; onlineServers=0 2024-11-24T04:26:12,202 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'fadb2fe0ba32,33969,1732422332893' ***** 2024-11-24T04:26:12,202 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-24T04:26:12,203 INFO [M:0;fadb2fe0ba32:33969 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-24T04:26:12,203 INFO [M:0;fadb2fe0ba32:33969 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-24T04:26:12,203 DEBUG [M:0;fadb2fe0ba32:33969 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-24T04:26:12,203 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-24T04:26:12,203 DEBUG [M:0;fadb2fe0ba32:33969 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-24T04:26:12,203 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster-HFileCleaner.large.0-1732422334783 {}] cleaner.HFileCleaner(306): Exit Thread[master/fadb2fe0ba32:0:becomeActiveMaster-HFileCleaner.large.0-1732422334783,5,FailOnTimeoutGroup] 2024-11-24T04:26:12,203 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster-HFileCleaner.small.0-1732422334784 {}] cleaner.HFileCleaner(306): Exit Thread[master/fadb2fe0ba32:0:becomeActiveMaster-HFileCleaner.small.0-1732422334784,5,FailOnTimeoutGroup] 2024-11-24T04:26:12,203 INFO [M:0;fadb2fe0ba32:33969 {}] hbase.ChoreService(370): Chore service for: master/fadb2fe0ba32:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-24T04:26:12,203 INFO [M:0;fadb2fe0ba32:33969 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-24T04:26:12,203 DEBUG [M:0;fadb2fe0ba32:33969 {}] master.HMaster(1795): Stopping service threads 2024-11-24T04:26:12,203 INFO [M:0;fadb2fe0ba32:33969 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-24T04:26:12,204 INFO [M:0;fadb2fe0ba32:33969 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-24T04:26:12,204 INFO [M:0;fadb2fe0ba32:33969 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-24T04:26:12,204 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-24T04:26:12,206 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33969-0x1014838d9450000, quorum=127.0.0.1:62017, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-24T04:26:12,206 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33969-0x1014838d9450000, quorum=127.0.0.1:62017, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T04:26:12,206 DEBUG [M:0;fadb2fe0ba32:33969 {}] zookeeper.ZKUtil(347): master:33969-0x1014838d9450000, quorum=127.0.0.1:62017, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-24T04:26:12,207 WARN [M:0;fadb2fe0ba32:33969 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-24T04:26:12,207 INFO [M:0;fadb2fe0ba32:33969 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:44611/user/jenkins/test-data/33c38499-d2fe-b6f0-8d8b-5809759afd19/.lastflushedseqids 2024-11-24T04:26:12,219 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44781 is added to blk_1073741941_1121 (size=138) 2024-11-24T04:26:12,219 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36069 is added to blk_1073741941_1121 (size=138) 2024-11-24T04:26:12,220 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34457 is added to blk_1073741941_1121 (size=138) 2024-11-24T04:26:12,220 INFO [M:0;fadb2fe0ba32:33969 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-24T04:26:12,220 INFO [M:0;fadb2fe0ba32:33969 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-24T04:26:12,221 DEBUG [M:0;fadb2fe0ba32:33969 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-24T04:26:12,221 INFO [M:0;fadb2fe0ba32:33969 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T04:26:12,221 DEBUG [M:0;fadb2fe0ba32:33969 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T04:26:12,221 DEBUG [M:0;fadb2fe0ba32:33969 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-24T04:26:12,221 DEBUG [M:0;fadb2fe0ba32:33969 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T04:26:12,221 INFO [M:0;fadb2fe0ba32:33969 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=68.34 KB heapSize=83.73 KB 2024-11-24T04:26:12,236 DEBUG [M:0;fadb2fe0ba32:33969 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44611/user/jenkins/test-data/33c38499-d2fe-b6f0-8d8b-5809759afd19/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/2fbb89b956df4c0dac9533bd8de15134 is 82, key is hbase:meta,,1/info:regioninfo/1732422335653/Put/seqid=0 2024-11-24T04:26:12,242 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36069 is added to blk_1073741942_1122 (size=5672) 2024-11-24T04:26:12,242 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34457 is added to blk_1073741942_1122 (size=5672) 2024-11-24T04:26:12,242 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44781 is added to blk_1073741942_1122 (size=5672) 2024-11-24T04:26:12,243 INFO [M:0;fadb2fe0ba32:33969 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:44611/user/jenkins/test-data/33c38499-d2fe-b6f0-8d8b-5809759afd19/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/2fbb89b956df4c0dac9533bd8de15134 2024-11-24T04:26:12,262 DEBUG [M:0;fadb2fe0ba32:33969 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44611/user/jenkins/test-data/33c38499-d2fe-b6f0-8d8b-5809759afd19/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/387ca780e765426ca8f53400710af9df is 1076, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1732422349232/Put/seqid=0 2024-11-24T04:26:12,267 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44781 is added to blk_1073741943_1123 (size=7755) 2024-11-24T04:26:12,267 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34457 is added to blk_1073741943_1123 (size=7755) 2024-11-24T04:26:12,268 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36069 is added to blk_1073741943_1123 (size=7755) 2024-11-24T04:26:12,268 INFO [M:0;fadb2fe0ba32:33969 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=67.61 KB at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:44611/user/jenkins/test-data/33c38499-d2fe-b6f0-8d8b-5809759afd19/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/387ca780e765426ca8f53400710af9df 2024-11-24T04:26:12,273 INFO [M:0;fadb2fe0ba32:33969 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 387ca780e765426ca8f53400710af9df 2024-11-24T04:26:12,287 DEBUG [M:0;fadb2fe0ba32:33969 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44611/user/jenkins/test-data/33c38499-d2fe-b6f0-8d8b-5809759afd19/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/d30c87d227104069bbe59587795541a0 is 69, key is fadb2fe0ba32,35977,1732422333601/rs:state/1732422334963/Put/seqid=0 2024-11-24T04:26:12,293 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44781 is added to blk_1073741944_1124 (size=5445) 2024-11-24T04:26:12,293 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34457 is added to blk_1073741944_1124 (size=5445) 2024-11-24T04:26:12,293 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36069 is added to blk_1073741944_1124 (size=5445) 2024-11-24T04:26:12,294 INFO [M:0;fadb2fe0ba32:33969 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=249 B at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:44611/user/jenkins/test-data/33c38499-d2fe-b6f0-8d8b-5809759afd19/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/d30c87d227104069bbe59587795541a0 2024-11-24T04:26:12,298 INFO [M:0;fadb2fe0ba32:33969 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for d30c87d227104069bbe59587795541a0 2024-11-24T04:26:12,299 DEBUG [M:0;fadb2fe0ba32:33969 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44611/user/jenkins/test-data/33c38499-d2fe-b6f0-8d8b-5809759afd19/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/2fbb89b956df4c0dac9533bd8de15134 as hdfs://localhost:44611/user/jenkins/test-data/33c38499-d2fe-b6f0-8d8b-5809759afd19/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/2fbb89b956df4c0dac9533bd8de15134 2024-11-24T04:26:12,301 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35977-0x1014838d9450001, quorum=127.0.0.1:62017, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-24T04:26:12,301 INFO [RS:0;fadb2fe0ba32:35977 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-24T04:26:12,301 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35977-0x1014838d9450001, quorum=127.0.0.1:62017, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-24T04:26:12,301 INFO [RS:0;fadb2fe0ba32:35977 {}] regionserver.HRegionServer(1031): Exiting; stopping=fadb2fe0ba32,35977,1732422333601; zookeeper connection closed. 2024-11-24T04:26:12,302 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@583c9bd7 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@583c9bd7 2024-11-24T04:26:12,302 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 3 regionserver(s) complete 2024-11-24T04:26:12,303 INFO [M:0;fadb2fe0ba32:33969 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44611/user/jenkins/test-data/33c38499-d2fe-b6f0-8d8b-5809759afd19/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/2fbb89b956df4c0dac9533bd8de15134, entries=8, sequenceid=168, filesize=5.5 K 2024-11-24T04:26:12,304 DEBUG [M:0;fadb2fe0ba32:33969 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44611/user/jenkins/test-data/33c38499-d2fe-b6f0-8d8b-5809759afd19/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/387ca780e765426ca8f53400710af9df as hdfs://localhost:44611/user/jenkins/test-data/33c38499-d2fe-b6f0-8d8b-5809759afd19/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/387ca780e765426ca8f53400710af9df 2024-11-24T04:26:12,308 INFO [M:0;fadb2fe0ba32:33969 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 387ca780e765426ca8f53400710af9df 2024-11-24T04:26:12,308 INFO [M:0;fadb2fe0ba32:33969 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44611/user/jenkins/test-data/33c38499-d2fe-b6f0-8d8b-5809759afd19/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/387ca780e765426ca8f53400710af9df, entries=17, sequenceid=168, filesize=7.6 K 2024-11-24T04:26:12,309 DEBUG [M:0;fadb2fe0ba32:33969 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44611/user/jenkins/test-data/33c38499-d2fe-b6f0-8d8b-5809759afd19/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/d30c87d227104069bbe59587795541a0 as hdfs://localhost:44611/user/jenkins/test-data/33c38499-d2fe-b6f0-8d8b-5809759afd19/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/d30c87d227104069bbe59587795541a0 2024-11-24T04:26:12,313 INFO [M:0;fadb2fe0ba32:33969 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for d30c87d227104069bbe59587795541a0 2024-11-24T04:26:12,313 INFO [M:0;fadb2fe0ba32:33969 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44611/user/jenkins/test-data/33c38499-d2fe-b6f0-8d8b-5809759afd19/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/d30c87d227104069bbe59587795541a0, entries=3, sequenceid=168, filesize=5.3 K 2024-11-24T04:26:12,314 INFO [M:0;fadb2fe0ba32:33969 {}] regionserver.HRegion(3140): Finished flush of dataSize ~68.34 KB/69984, heapSize ~83.43 KB/85432, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 93ms, sequenceid=168, compaction requested=false 2024-11-24T04:26:12,315 INFO [M:0;fadb2fe0ba32:33969 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T04:26:12,316 DEBUG [M:0;fadb2fe0ba32:33969 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732422372221Disabling compacts and flushes for region at 1732422372221Disabling writes for close at 1732422372221Obtaining lock to block concurrent updates at 1732422372221Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732422372221Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=69984, getHeapSize=85672, getOffHeapSize=0, getCellsCount=195 at 1732422372221Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732422372222 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732422372222Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732422372236 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732422372236Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732422372247 (+11 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732422372261 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732422372261Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732422372273 (+12 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732422372287 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732422372287Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2067b9ac: reopening flushed file at 1732422372298 (+11 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@49313f30: reopening flushed file at 1732422372303 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@62b948b3: reopening flushed file at 1732422372308 (+5 ms)Finished flush of dataSize ~68.34 KB/69984, heapSize ~83.43 KB/85432, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 93ms, sequenceid=168, compaction requested=false at 1732422372314 (+6 ms)Writing region close event to WAL at 1732422372315 (+1 ms)Closed at 1732422372315 2024-11-24T04:26:12,318 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44781 is added to blk_1073741830_1006 (size=56409) 2024-11-24T04:26:12,318 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34457 is added to blk_1073741830_1006 (size=56409) 2024-11-24T04:26:12,318 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36069 is added to blk_1073741830_1006 (size=56409) 2024-11-24T04:26:12,318 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-24T04:26:12,319 INFO [M:0;fadb2fe0ba32:33969 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-24T04:26:12,319 INFO [M:0;fadb2fe0ba32:33969 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:33969 2024-11-24T04:26:12,319 INFO [M:0;fadb2fe0ba32:33969 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-24T04:26:12,421 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33969-0x1014838d9450000, quorum=127.0.0.1:62017, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-24T04:26:12,421 INFO [M:0;fadb2fe0ba32:33969 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-24T04:26:12,421 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33969-0x1014838d9450000, quorum=127.0.0.1:62017, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-24T04:26:12,427 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732422352259/wal.1732422352616 with renewLeaseKey: DEFAULT_16655 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732422352259/wal.1732422352616 (inode 16655) Holder DFSClient_NONMAPREDUCE_-1575425110_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732422352259/wal.1732422352616 (inode 16655) Holder DFSClient_NONMAPREDUCE_-1575425110_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy45.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy46.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-11-24T04:26:12,428 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732422344447/wal.1732422344535 with renewLeaseKey: DEFAULT_16586 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T04:26:12,428 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732422367306/wal.1732422367491 with renewLeaseKey: DEFAULT_16767 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T04:26:12,430 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732422352756/wal.1732422362031 with renewLeaseKey: DEFAULT_16678 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732422352756/wal.1732422362031 (inode 16678) Holder DFSClient_NONMAPREDUCE_-1575425110_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732422352756/wal.1732422362031 (inode 16678) Holder DFSClient_NONMAPREDUCE_-1575425110_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy45.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy46.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-11-24T04:26:12,431 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732422336789/wal.1732422336862 with renewLeaseKey: DEFAULT_16506 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T04:26:12,433 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testdatalosswheninputerror-manual,16010,1732422362236/wal.1732422362696 with renewLeaseKey: DEFAULT_16704 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testdatalosswheninputerror-manual,16010,1732422362236/wal.1732422362696 (inode 16704) Holder DFSClient_NONMAPREDUCE_-1575425110_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testdatalosswheninputerror-manual,16010,1732422362236/wal.1732422362696 (inode 16704) Holder DFSClient_NONMAPREDUCE_-1575425110_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy45.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy46.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-11-24T04:26:12,433 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732422362860/wal.1732422362917 with renewLeaseKey: DEFAULT_16726 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T04:26:12,435 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testnameconflictwhensplit1-manual,16010,1732422336482/wal.1732422336661 with renewLeaseKey: DEFAULT_16485 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testnameconflictwhensplit1-manual,16010,1732422336482/wal.1732422336661 (inode 16485) Holder DFSClient_NONMAPREDUCE_-1575425110_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testnameconflictwhensplit1-manual,16010,1732422336482/wal.1732422336661 (inode 16485) Holder DFSClient_NONMAPREDUCE_-1575425110_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy45.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy46.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-11-24T04:26:12,438 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testnameconflictwhensplit0-manual,16010,1732422335998/wal.1732422336225 with renewLeaseKey: DEFAULT_16462 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testnameconflictwhensplit0-manual,16010,1732422335998/wal.1732422336225 (inode 16462) Holder DFSClient_NONMAPREDUCE_-1575425110_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testnameconflictwhensplit0-manual,16010,1732422335998/wal.1732422336225 (inode 16462) Holder DFSClient_NONMAPREDUCE_-1575425110_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy45.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy46.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-11-24T04:26:12,443 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@35f1150e{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-24T04:26:12,445 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@13a77e13{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-24T04:26:12,445 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-24T04:26:12,446 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@f9972d0{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-24T04:26:12,446 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@2a6d5e13{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6816aaf7-45a5-053e-0edc-4c140671afba/hadoop.log.dir/,STOPPED} 2024-11-24T04:26:12,449 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-24T04:26:12,449 WARN [BP-1091946017-172.17.0.2-1732422329789 heartbeating to localhost/127.0.0.1:44611 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-24T04:26:12,449 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-24T04:26:12,449 WARN [BP-1091946017-172.17.0.2-1732422329789 heartbeating to localhost/127.0.0.1:44611 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1091946017-172.17.0.2-1732422329789 (Datanode Uuid d070398a-b698-4d3a-9f67-a5eac0f5f265) service to localhost/127.0.0.1:44611 2024-11-24T04:26:12,450 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6816aaf7-45a5-053e-0edc-4c140671afba/cluster_be0b9e06-9b41-befd-1079-43744c8c01cc/data/data5/current/BP-1091946017-172.17.0.2-1732422329789 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-24T04:26:12,450 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6816aaf7-45a5-053e-0edc-4c140671afba/cluster_be0b9e06-9b41-befd-1079-43744c8c01cc/data/data6/current/BP-1091946017-172.17.0.2-1732422329789 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-24T04:26:12,451 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-24T04:26:12,453 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@7bd427b8{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-24T04:26:12,453 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@6915083f{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-24T04:26:12,453 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-24T04:26:12,453 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@5cc2d6b{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-24T04:26:12,453 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@46b092e1{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6816aaf7-45a5-053e-0edc-4c140671afba/hadoop.log.dir/,STOPPED} 2024-11-24T04:26:12,455 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-24T04:26:12,455 WARN [BP-1091946017-172.17.0.2-1732422329789 heartbeating to localhost/127.0.0.1:44611 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-24T04:26:12,455 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-24T04:26:12,455 WARN [BP-1091946017-172.17.0.2-1732422329789 heartbeating to localhost/127.0.0.1:44611 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1091946017-172.17.0.2-1732422329789 (Datanode Uuid 60946a18-8e33-40fa-966b-81f112b81c54) service to localhost/127.0.0.1:44611 2024-11-24T04:26:12,455 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6816aaf7-45a5-053e-0edc-4c140671afba/cluster_be0b9e06-9b41-befd-1079-43744c8c01cc/data/data3/current/BP-1091946017-172.17.0.2-1732422329789 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-24T04:26:12,455 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-24T04:26:12,455 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6816aaf7-45a5-053e-0edc-4c140671afba/cluster_be0b9e06-9b41-befd-1079-43744c8c01cc/data/data4/current/BP-1091946017-172.17.0.2-1732422329789 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-24T04:26:12,461 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@330740de{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-24T04:26:12,461 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@7b24cab9{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-24T04:26:12,461 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-24T04:26:12,461 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4a359997{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-24T04:26:12,461 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@cf5a85e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6816aaf7-45a5-053e-0edc-4c140671afba/hadoop.log.dir/,STOPPED} 2024-11-24T04:26:12,462 WARN [BP-1091946017-172.17.0.2-1732422329789 heartbeating to localhost/127.0.0.1:44611 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-24T04:26:12,463 WARN [BP-1091946017-172.17.0.2-1732422329789 heartbeating to localhost/127.0.0.1:44611 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1091946017-172.17.0.2-1732422329789 (Datanode Uuid c4648929-9bd7-4a94-827d-36f0dcfe9f17) service to localhost/127.0.0.1:44611 2024-11-24T04:26:12,463 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-24T04:26:12,463 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-24T04:26:12,463 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6816aaf7-45a5-053e-0edc-4c140671afba/cluster_be0b9e06-9b41-befd-1079-43744c8c01cc/data/data1/current/BP-1091946017-172.17.0.2-1732422329789 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-24T04:26:12,463 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6816aaf7-45a5-053e-0edc-4c140671afba/cluster_be0b9e06-9b41-befd-1079-43744c8c01cc/data/data2/current/BP-1091946017-172.17.0.2-1732422329789 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-24T04:26:12,464 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-24T04:26:12,471 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@3717288f{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-24T04:26:12,472 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@4bd70930{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-24T04:26:12,472 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-24T04:26:12,472 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6dc9d5c1{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-24T04:26:12,472 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4f37ffca{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6816aaf7-45a5-053e-0edc-4c140671afba/hadoop.log.dir/,STOPPED} 2024-11-24T04:26:12,482 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-24T04:26:12,530 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down