2024-12-03 15:56:52,459 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@20b2475a 2024-12-03 15:56:52,482 main DEBUG Took 0.020620 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-12-03 15:56:52,484 main DEBUG PluginManager 'Core' found 129 plugins 2024-12-03 15:56:52,484 main DEBUG PluginManager 'Level' found 0 plugins 2024-12-03 15:56:52,486 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-12-03 15:56:52,487 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 15:56:52,496 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-12-03 15:56:52,509 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 15:56:52,511 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 15:56:52,511 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 15:56:52,512 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 15:56:52,513 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 15:56:52,513 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 15:56:52,514 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 15:56:52,515 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 15:56:52,515 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 15:56:52,516 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 15:56:52,517 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 15:56:52,517 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 15:56:52,518 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 15:56:52,518 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 15:56:52,519 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 15:56:52,519 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 15:56:52,520 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 15:56:52,520 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 15:56:52,521 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 15:56:52,521 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 15:56:52,522 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 15:56:52,522 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 15:56:52,523 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 15:56:52,523 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 15:56:52,524 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 15:56:52,524 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-12-03 15:56:52,526 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 15:56:52,528 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-12-03 15:56:52,531 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-12-03 15:56:52,531 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-12-03 15:56:52,533 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-12-03 15:56:52,533 main DEBUG PluginManager 'Converter' found 47 plugins 2024-12-03 15:56:52,543 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-12-03 15:56:52,546 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-12-03 15:56:52,548 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-12-03 15:56:52,549 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-12-03 15:56:52,549 main DEBUG createAppenders(={Console}) 2024-12-03 15:56:52,550 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@20b2475a initialized 2024-12-03 15:56:52,550 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@20b2475a 2024-12-03 15:56:52,551 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@20b2475a OK. 2024-12-03 15:56:52,552 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-12-03 15:56:52,552 main DEBUG OutputStream closed 2024-12-03 15:56:52,552 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-12-03 15:56:52,553 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-12-03 15:56:52,553 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@4310d43 OK 2024-12-03 15:56:52,636 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-12-03 15:56:52,639 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-12-03 15:56:52,640 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-12-03 15:56:52,641 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-12-03 15:56:52,642 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-12-03 15:56:52,643 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-12-03 15:56:52,643 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-12-03 15:56:52,643 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-12-03 15:56:52,644 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-12-03 15:56:52,644 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-12-03 15:56:52,645 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-12-03 15:56:52,645 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-12-03 15:56:52,646 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-12-03 15:56:52,646 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-12-03 15:56:52,646 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-12-03 15:56:52,647 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-12-03 15:56:52,647 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-12-03 15:56:52,648 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-12-03 15:56:52,650 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-03 15:56:52,650 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@12f9af83) with optional ClassLoader: null 2024-12-03 15:56:52,651 main DEBUG Shutdown hook enabled. Registering a new one. 2024-12-03 15:56:52,651 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@12f9af83] started OK. 2024-12-03T15:56:52,919 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb3fe2df-6726-509d-596d-17caab0e5978 2024-12-03 15:56:52,922 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-12-03 15:56:52,923 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-03T15:56:52,932 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay timeout: 13 mins 2024-12-03T15:56:52,937 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.regionserver.wal.TestSecureAsyncWALReplay timeout: 13 mins 2024-12-03T15:56:52,957 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-12-03T15:56:53,002 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-12-03T15:56:53,002 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-12-03T15:56:53,014 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=3, rsPorts=, rsClass=null, numDataNodes=3, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-12-03T15:56:53,030 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb3fe2df-6726-509d-596d-17caab0e5978/cluster_ef20cb83-4e1e-0871-5945-558338f0ff60, deleteOnExit=true 2024-12-03T15:56:53,031 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-12-03T15:56:53,032 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb3fe2df-6726-509d-596d-17caab0e5978/test.cache.data in system properties and HBase conf 2024-12-03T15:56:53,033 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb3fe2df-6726-509d-596d-17caab0e5978/hadoop.tmp.dir in system properties and HBase conf 2024-12-03T15:56:53,034 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb3fe2df-6726-509d-596d-17caab0e5978/hadoop.log.dir in system properties and HBase conf 2024-12-03T15:56:53,034 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb3fe2df-6726-509d-596d-17caab0e5978/mapreduce.cluster.local.dir in system properties and HBase conf 2024-12-03T15:56:53,035 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb3fe2df-6726-509d-596d-17caab0e5978/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-12-03T15:56:53,036 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-12-03T15:56:53,122 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-12-03T15:56:53,216 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-03T15:56:53,220 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb3fe2df-6726-509d-596d-17caab0e5978/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-12-03T15:56:53,220 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb3fe2df-6726-509d-596d-17caab0e5978/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-12-03T15:56:53,220 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb3fe2df-6726-509d-596d-17caab0e5978/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-12-03T15:56:53,221 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb3fe2df-6726-509d-596d-17caab0e5978/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-03T15:56:53,222 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb3fe2df-6726-509d-596d-17caab0e5978/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-12-03T15:56:53,222 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb3fe2df-6726-509d-596d-17caab0e5978/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-12-03T15:56:53,222 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb3fe2df-6726-509d-596d-17caab0e5978/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-03T15:56:53,223 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb3fe2df-6726-509d-596d-17caab0e5978/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-03T15:56:53,223 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb3fe2df-6726-509d-596d-17caab0e5978/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-12-03T15:56:53,223 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb3fe2df-6726-509d-596d-17caab0e5978/nfs.dump.dir in system properties and HBase conf 2024-12-03T15:56:53,224 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb3fe2df-6726-509d-596d-17caab0e5978/java.io.tmpdir in system properties and HBase conf 2024-12-03T15:56:53,224 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb3fe2df-6726-509d-596d-17caab0e5978/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-03T15:56:53,224 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb3fe2df-6726-509d-596d-17caab0e5978/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-12-03T15:56:53,225 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb3fe2df-6726-509d-596d-17caab0e5978/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-12-03T15:56:54,064 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-12-03T15:56:54,146 INFO [Time-limited test {}] log.Log(170): Logging initialized @2827ms to org.eclipse.jetty.util.log.Slf4jLog 2024-12-03T15:56:54,222 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-03T15:56:54,289 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-03T15:56:54,308 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-03T15:56:54,309 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-03T15:56:54,310 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-03T15:56:54,330 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-03T15:56:54,332 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@264e890{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb3fe2df-6726-509d-596d-17caab0e5978/hadoop.log.dir/,AVAILABLE} 2024-12-03T15:56:54,333 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@652099c6{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-03T15:56:54,547 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4c03e044{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb3fe2df-6726-509d-596d-17caab0e5978/java.io.tmpdir/jetty-localhost-44003-hadoop-hdfs-3_4_1-tests_jar-_-any-9335717068046576622/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-03T15:56:54,554 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@698721f1{HTTP/1.1, (http/1.1)}{localhost:44003} 2024-12-03T15:56:54,554 INFO [Time-limited test {}] server.Server(415): Started @3236ms 2024-12-03T15:56:54,941 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-03T15:56:54,948 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-03T15:56:54,949 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-03T15:56:54,949 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-03T15:56:54,950 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-03T15:56:54,950 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3161f7a1{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb3fe2df-6726-509d-596d-17caab0e5978/hadoop.log.dir/,AVAILABLE} 2024-12-03T15:56:54,951 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@751d2fa4{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-03T15:56:55,075 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@501095cc{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb3fe2df-6726-509d-596d-17caab0e5978/java.io.tmpdir/jetty-localhost-36349-hadoop-hdfs-3_4_1-tests_jar-_-any-106379417895317026/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-03T15:56:55,076 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@2b37aa87{HTTP/1.1, (http/1.1)}{localhost:36349} 2024-12-03T15:56:55,076 INFO [Time-limited test {}] server.Server(415): Started @3758ms 2024-12-03T15:56:55,129 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-03T15:56:55,288 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-03T15:56:55,297 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-03T15:56:55,309 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-03T15:56:55,309 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-03T15:56:55,309 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-03T15:56:55,312 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7acb2c1e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb3fe2df-6726-509d-596d-17caab0e5978/hadoop.log.dir/,AVAILABLE} 2024-12-03T15:56:55,313 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7c7d32f8{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-03T15:56:55,486 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@64052af6{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb3fe2df-6726-509d-596d-17caab0e5978/java.io.tmpdir/jetty-localhost-38769-hadoop-hdfs-3_4_1-tests_jar-_-any-801083287870729883/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-03T15:56:55,487 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@645d24a7{HTTP/1.1, (http/1.1)}{localhost:38769} 2024-12-03T15:56:55,488 INFO [Time-limited test {}] server.Server(415): Started @4169ms 2024-12-03T15:56:55,491 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-03T15:56:55,621 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-03T15:56:55,627 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-03T15:56:55,643 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-03T15:56:55,644 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-03T15:56:55,644 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-12-03T15:56:55,646 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1e1a8b20{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb3fe2df-6726-509d-596d-17caab0e5978/hadoop.log.dir/,AVAILABLE} 2024-12-03T15:56:55,647 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@228ffa29{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-03T15:56:55,693 WARN [Thread-105 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb3fe2df-6726-509d-596d-17caab0e5978/cluster_ef20cb83-4e1e-0871-5945-558338f0ff60/data/data3/current/BP-1122350750-172.17.0.3-1733241413817/current, will proceed with Du for space computation calculation, 2024-12-03T15:56:55,693 WARN [Thread-106 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb3fe2df-6726-509d-596d-17caab0e5978/cluster_ef20cb83-4e1e-0871-5945-558338f0ff60/data/data1/current/BP-1122350750-172.17.0.3-1733241413817/current, will proceed with Du for space computation calculation, 2024-12-03T15:56:55,693 WARN [Thread-107 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb3fe2df-6726-509d-596d-17caab0e5978/cluster_ef20cb83-4e1e-0871-5945-558338f0ff60/data/data4/current/BP-1122350750-172.17.0.3-1733241413817/current, will proceed with Du for space computation calculation, 2024-12-03T15:56:55,693 WARN [Thread-108 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb3fe2df-6726-509d-596d-17caab0e5978/cluster_ef20cb83-4e1e-0871-5945-558338f0ff60/data/data2/current/BP-1122350750-172.17.0.3-1733241413817/current, will proceed with Du for space computation calculation, 2024-12-03T15:56:55,789 WARN [Thread-83 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-03T15:56:55,791 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-03T15:56:55,818 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@420bc738{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb3fe2df-6726-509d-596d-17caab0e5978/java.io.tmpdir/jetty-localhost-36077-hadoop-hdfs-3_4_1-tests_jar-_-any-10494369471116926258/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-03T15:56:55,819 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@5546381a{HTTP/1.1, (http/1.1)}{localhost:36077} 2024-12-03T15:56:55,819 INFO [Time-limited test {}] server.Server(415): Started @4501ms 2024-12-03T15:56:55,821 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-03T15:56:55,873 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xe0711300560062fb with lease ID 0x4208025399660e98: Processing first storage report for DS-e007f1cc-fc1b-4368-9716-e11e4fe2f4eb from datanode DatanodeRegistration(127.0.0.1:44827, datanodeUuid=98c3b8bb-8b94-459f-a46e-c24ee437a325, infoPort=46731, infoSecurePort=0, ipcPort=35763, storageInfo=lv=-57;cid=testClusterID;nsid=1962606829;c=1733241413817) 2024-12-03T15:56:55,874 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xe0711300560062fb with lease ID 0x4208025399660e98: from storage DS-e007f1cc-fc1b-4368-9716-e11e4fe2f4eb node DatanodeRegistration(127.0.0.1:44827, datanodeUuid=98c3b8bb-8b94-459f-a46e-c24ee437a325, infoPort=46731, infoSecurePort=0, ipcPort=35763, storageInfo=lv=-57;cid=testClusterID;nsid=1962606829;c=1733241413817), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-12-03T15:56:55,874 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x74220671e414ffd6 with lease ID 0x4208025399660e99: Processing first storage report for DS-5c6771fd-75c7-4775-ad88-ae5bbe88de5c from datanode DatanodeRegistration(127.0.0.1:46585, datanodeUuid=b7a8f304-74d8-4a03-ac93-6ac0859d29b6, infoPort=37355, infoSecurePort=0, ipcPort=34351, storageInfo=lv=-57;cid=testClusterID;nsid=1962606829;c=1733241413817) 2024-12-03T15:56:55,875 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x74220671e414ffd6 with lease ID 0x4208025399660e99: from storage DS-5c6771fd-75c7-4775-ad88-ae5bbe88de5c node DatanodeRegistration(127.0.0.1:46585, datanodeUuid=b7a8f304-74d8-4a03-ac93-6ac0859d29b6, infoPort=37355, infoSecurePort=0, ipcPort=34351, storageInfo=lv=-57;cid=testClusterID;nsid=1962606829;c=1733241413817), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-03T15:56:55,875 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xe0711300560062fb with lease ID 0x4208025399660e98: Processing first storage report for DS-24fe3433-a04c-43f9-854e-09201190c2ce from datanode DatanodeRegistration(127.0.0.1:44827, datanodeUuid=98c3b8bb-8b94-459f-a46e-c24ee437a325, infoPort=46731, infoSecurePort=0, ipcPort=35763, storageInfo=lv=-57;cid=testClusterID;nsid=1962606829;c=1733241413817) 2024-12-03T15:56:55,875 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xe0711300560062fb with lease ID 0x4208025399660e98: from storage DS-24fe3433-a04c-43f9-854e-09201190c2ce node DatanodeRegistration(127.0.0.1:44827, datanodeUuid=98c3b8bb-8b94-459f-a46e-c24ee437a325, infoPort=46731, infoSecurePort=0, ipcPort=35763, storageInfo=lv=-57;cid=testClusterID;nsid=1962606829;c=1733241413817), blocks: 0, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0 2024-12-03T15:56:55,876 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x74220671e414ffd6 with lease ID 0x4208025399660e99: Processing first storage report for DS-157a9707-d23f-47d9-96a4-b6c44f977dc3 from datanode DatanodeRegistration(127.0.0.1:46585, datanodeUuid=b7a8f304-74d8-4a03-ac93-6ac0859d29b6, infoPort=37355, infoSecurePort=0, ipcPort=34351, storageInfo=lv=-57;cid=testClusterID;nsid=1962606829;c=1733241413817) 2024-12-03T15:56:55,876 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x74220671e414ffd6 with lease ID 0x4208025399660e99: from storage DS-157a9707-d23f-47d9-96a4-b6c44f977dc3 node DatanodeRegistration(127.0.0.1:46585, datanodeUuid=b7a8f304-74d8-4a03-ac93-6ac0859d29b6, infoPort=37355, infoSecurePort=0, ipcPort=34351, storageInfo=lv=-57;cid=testClusterID;nsid=1962606829;c=1733241413817), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-03T15:56:56,004 WARN [Thread-139 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb3fe2df-6726-509d-596d-17caab0e5978/cluster_ef20cb83-4e1e-0871-5945-558338f0ff60/data/data5/current/BP-1122350750-172.17.0.3-1733241413817/current, will proceed with Du for space computation calculation, 2024-12-03T15:56:56,007 WARN [Thread-140 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb3fe2df-6726-509d-596d-17caab0e5978/cluster_ef20cb83-4e1e-0871-5945-558338f0ff60/data/data6/current/BP-1122350750-172.17.0.3-1733241413817/current, will proceed with Du for space computation calculation, 2024-12-03T15:56:56,055 WARN [Thread-129 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-03T15:56:56,070 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x394eec01d294355b with lease ID 0x4208025399660e9a: Processing first storage report for DS-309fe9b7-1997-44cb-b0c9-1ce3c04a71de from datanode DatanodeRegistration(127.0.0.1:40575, datanodeUuid=5360fc40-fc97-40f7-a77d-73c889ffe94e, infoPort=44513, infoSecurePort=0, ipcPort=45961, storageInfo=lv=-57;cid=testClusterID;nsid=1962606829;c=1733241413817) 2024-12-03T15:56:56,070 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x394eec01d294355b with lease ID 0x4208025399660e9a: from storage DS-309fe9b7-1997-44cb-b0c9-1ce3c04a71de node DatanodeRegistration(127.0.0.1:40575, datanodeUuid=5360fc40-fc97-40f7-a77d-73c889ffe94e, infoPort=44513, infoSecurePort=0, ipcPort=45961, storageInfo=lv=-57;cid=testClusterID;nsid=1962606829;c=1733241413817), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-12-03T15:56:56,071 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x394eec01d294355b with lease ID 0x4208025399660e9a: Processing first storage report for DS-65246ce4-cc68-4b88-9395-d3543e98fddb from datanode DatanodeRegistration(127.0.0.1:40575, datanodeUuid=5360fc40-fc97-40f7-a77d-73c889ffe94e, infoPort=44513, infoSecurePort=0, ipcPort=45961, storageInfo=lv=-57;cid=testClusterID;nsid=1962606829;c=1733241413817) 2024-12-03T15:56:56,072 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x394eec01d294355b with lease ID 0x4208025399660e9a: from storage DS-65246ce4-cc68-4b88-9395-d3543e98fddb node DatanodeRegistration(127.0.0.1:40575, datanodeUuid=5360fc40-fc97-40f7-a77d-73c889ffe94e, infoPort=44513, infoSecurePort=0, ipcPort=45961, storageInfo=lv=-57;cid=testClusterID;nsid=1962606829;c=1733241413817), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-03T15:56:56,284 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb3fe2df-6726-509d-596d-17caab0e5978 2024-12-03T15:56:56,373 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb3fe2df-6726-509d-596d-17caab0e5978/cluster_ef20cb83-4e1e-0871-5945-558338f0ff60/zookeeper_0, clientPort=61024, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb3fe2df-6726-509d-596d-17caab0e5978/cluster_ef20cb83-4e1e-0871-5945-558338f0ff60/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb3fe2df-6726-509d-596d-17caab0e5978/cluster_ef20cb83-4e1e-0871-5945-558338f0ff60/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-12-03T15:56:56,388 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=61024 2024-12-03T15:56:56,424 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-03T15:56:56,428 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-03T15:56:56,705 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46585 is added to blk_1073741825_1001 (size=7) 2024-12-03T15:56:56,706 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44827 is added to blk_1073741825_1001 (size=7) 2024-12-03T15:56:56,706 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40575 is added to blk_1073741825_1001 (size=7) 2024-12-03T15:56:57,118 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:45015/user/jenkins/test-data/c81ce3c5-e5b8-1c2f-869e-120d734cf311 with version=8 2024-12-03T15:56:57,118 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:45015/user/jenkins/test-data/c81ce3c5-e5b8-1c2f-869e-120d734cf311/hbase-staging 2024-12-03T15:56:57,476 INFO [Time-limited test {}] client.ConnectionUtils(128): master/713339a81dd9:0 server-side Connection retries=45 2024-12-03T15:56:57,489 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-03T15:56:57,490 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-03T15:56:57,496 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-03T15:56:57,496 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-03T15:56:57,497 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-03T15:56:57,697 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-12-03T15:56:57,788 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-12-03T15:56:57,804 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-12-03T15:56:57,810 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-03T15:56:57,849 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 32706 (auto-detected) 2024-12-03T15:56:57,850 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:03 (auto-detected) 2024-12-03T15:56:57,880 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:34663 2024-12-03T15:56:57,910 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:34663 connecting to ZooKeeper ensemble=127.0.0.1:61024 2024-12-03T15:56:57,956 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:346630x0, quorum=127.0.0.1:61024, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-03T15:56:57,968 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:34663-0x1017926911e0000 connected 2024-12-03T15:56:58,015 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-03T15:56:58,020 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-03T15:56:58,035 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:34663-0x1017926911e0000, quorum=127.0.0.1:61024, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-03T15:56:58,041 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:45015/user/jenkins/test-data/c81ce3c5-e5b8-1c2f-869e-120d734cf311, hbase.cluster.distributed=false 2024-12-03T15:56:58,072 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:34663-0x1017926911e0000, quorum=127.0.0.1:61024, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-03T15:56:58,078 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=34663 2024-12-03T15:56:58,078 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=34663 2024-12-03T15:56:58,079 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=34663 2024-12-03T15:56:58,080 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=34663 2024-12-03T15:56:58,080 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=34663 2024-12-03T15:56:58,231 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/713339a81dd9:0 server-side Connection retries=45 2024-12-03T15:56:58,233 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-03T15:56:58,234 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-03T15:56:58,234 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-03T15:56:58,234 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-03T15:56:58,234 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-03T15:56:58,237 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-03T15:56:58,240 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-03T15:56:58,242 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:42147 2024-12-03T15:56:58,244 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:42147 connecting to ZooKeeper ensemble=127.0.0.1:61024 2024-12-03T15:56:58,246 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-03T15:56:58,249 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-03T15:56:58,262 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:421470x0, quorum=127.0.0.1:61024, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-03T15:56:58,262 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:421470x0, quorum=127.0.0.1:61024, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-03T15:56:58,263 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:42147-0x1017926911e0001 connected 2024-12-03T15:56:58,267 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-03T15:56:58,277 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-03T15:56:58,281 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:42147-0x1017926911e0001, quorum=127.0.0.1:61024, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-03T15:56:58,288 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:42147-0x1017926911e0001, quorum=127.0.0.1:61024, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-03T15:56:58,289 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=42147 2024-12-03T15:56:58,290 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=42147 2024-12-03T15:56:58,294 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=42147 2024-12-03T15:56:58,295 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=42147 2024-12-03T15:56:58,296 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=42147 2024-12-03T15:56:58,314 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/713339a81dd9:0 server-side Connection retries=45 2024-12-03T15:56:58,315 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-03T15:56:58,315 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-03T15:56:58,315 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-03T15:56:58,315 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-03T15:56:58,316 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-03T15:56:58,316 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-03T15:56:58,316 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-03T15:56:58,318 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:34819 2024-12-03T15:56:58,321 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:34819 connecting to ZooKeeper ensemble=127.0.0.1:61024 2024-12-03T15:56:58,322 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-03T15:56:58,327 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-03T15:56:58,339 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:348190x0, quorum=127.0.0.1:61024, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-03T15:56:58,340 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:34819-0x1017926911e0002 connected 2024-12-03T15:56:58,341 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:34819-0x1017926911e0002, quorum=127.0.0.1:61024, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-03T15:56:58,341 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-03T15:56:58,346 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-03T15:56:58,348 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:34819-0x1017926911e0002, quorum=127.0.0.1:61024, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-03T15:56:58,350 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:34819-0x1017926911e0002, quorum=127.0.0.1:61024, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-03T15:56:58,354 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=34819 2024-12-03T15:56:58,355 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=34819 2024-12-03T15:56:58,356 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=34819 2024-12-03T15:56:58,357 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=34819 2024-12-03T15:56:58,357 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=34819 2024-12-03T15:56:58,384 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/713339a81dd9:0 server-side Connection retries=45 2024-12-03T15:56:58,384 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-03T15:56:58,384 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-03T15:56:58,384 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-03T15:56:58,385 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-03T15:56:58,385 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-03T15:56:58,385 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-03T15:56:58,385 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-03T15:56:58,389 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:36387 2024-12-03T15:56:58,391 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:36387 connecting to ZooKeeper ensemble=127.0.0.1:61024 2024-12-03T15:56:58,393 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-03T15:56:58,396 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-03T15:56:58,412 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:363870x0, quorum=127.0.0.1:61024, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-03T15:56:58,414 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:363870x0, quorum=127.0.0.1:61024, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-03T15:56:58,415 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-03T15:56:58,416 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:36387-0x1017926911e0003 connected 2024-12-03T15:56:58,417 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-03T15:56:58,418 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36387-0x1017926911e0003, quorum=127.0.0.1:61024, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-03T15:56:58,421 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36387-0x1017926911e0003, quorum=127.0.0.1:61024, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-03T15:56:58,426 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=36387 2024-12-03T15:56:58,427 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=36387 2024-12-03T15:56:58,429 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=36387 2024-12-03T15:56:58,434 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=36387 2024-12-03T15:56:58,435 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=36387 2024-12-03T15:56:58,458 DEBUG [M:0;713339a81dd9:34663 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;713339a81dd9:34663 2024-12-03T15:56:58,461 INFO [master/713339a81dd9:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/713339a81dd9,34663,1733241417250 2024-12-03T15:56:58,470 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42147-0x1017926911e0001, quorum=127.0.0.1:61024, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-03T15:56:58,471 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36387-0x1017926911e0003, quorum=127.0.0.1:61024, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-03T15:56:58,471 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34819-0x1017926911e0002, quorum=127.0.0.1:61024, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-03T15:56:58,471 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34663-0x1017926911e0000, quorum=127.0.0.1:61024, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-03T15:56:58,477 DEBUG [master/713339a81dd9:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:34663-0x1017926911e0000, quorum=127.0.0.1:61024, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/713339a81dd9,34663,1733241417250 2024-12-03T15:56:58,508 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34819-0x1017926911e0002, quorum=127.0.0.1:61024, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-03T15:56:58,508 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36387-0x1017926911e0003, quorum=127.0.0.1:61024, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-03T15:56:58,509 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36387-0x1017926911e0003, quorum=127.0.0.1:61024, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T15:56:58,509 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34819-0x1017926911e0002, quorum=127.0.0.1:61024, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T15:56:58,510 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34663-0x1017926911e0000, quorum=127.0.0.1:61024, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T15:56:58,510 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42147-0x1017926911e0001, quorum=127.0.0.1:61024, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-03T15:56:58,510 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42147-0x1017926911e0001, quorum=127.0.0.1:61024, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T15:56:58,512 DEBUG [master/713339a81dd9:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:34663-0x1017926911e0000, quorum=127.0.0.1:61024, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-03T15:56:58,514 INFO [master/713339a81dd9:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/713339a81dd9,34663,1733241417250 from backup master directory 2024-12-03T15:56:58,517 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34663-0x1017926911e0000, quorum=127.0.0.1:61024, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/713339a81dd9,34663,1733241417250 2024-12-03T15:56:58,517 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34819-0x1017926911e0002, quorum=127.0.0.1:61024, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-03T15:56:58,517 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34663-0x1017926911e0000, quorum=127.0.0.1:61024, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-03T15:56:58,517 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36387-0x1017926911e0003, quorum=127.0.0.1:61024, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-03T15:56:58,517 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42147-0x1017926911e0001, quorum=127.0.0.1:61024, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-03T15:56:58,518 WARN [master/713339a81dd9:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-03T15:56:58,520 INFO [master/713339a81dd9:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=713339a81dd9,34663,1733241417250 2024-12-03T15:56:58,523 INFO [master/713339a81dd9:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-12-03T15:56:58,525 INFO [master/713339a81dd9:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-12-03T15:56:58,609 DEBUG [master/713339a81dd9:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:45015/user/jenkins/test-data/c81ce3c5-e5b8-1c2f-869e-120d734cf311/hbase.id] with ID: a4fd42ae-14ff-4383-be19-b09963b80309 2024-12-03T15:56:58,609 DEBUG [master/713339a81dd9:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:45015/user/jenkins/test-data/c81ce3c5-e5b8-1c2f-869e-120d734cf311/.tmp/hbase.id 2024-12-03T15:56:58,629 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44827 is added to blk_1073741826_1002 (size=42) 2024-12-03T15:56:58,629 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40575 is added to blk_1073741826_1002 (size=42) 2024-12-03T15:56:58,630 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46585 is added to blk_1073741826_1002 (size=42) 2024-12-03T15:56:58,631 DEBUG [master/713339a81dd9:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:45015/user/jenkins/test-data/c81ce3c5-e5b8-1c2f-869e-120d734cf311/.tmp/hbase.id]:[hdfs://localhost:45015/user/jenkins/test-data/c81ce3c5-e5b8-1c2f-869e-120d734cf311/hbase.id] 2024-12-03T15:56:58,679 INFO [master/713339a81dd9:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-03T15:56:58,685 INFO [master/713339a81dd9:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-12-03T15:56:58,715 INFO [master/713339a81dd9:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 27ms. 2024-12-03T15:56:58,721 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34819-0x1017926911e0002, quorum=127.0.0.1:61024, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T15:56:58,721 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42147-0x1017926911e0001, quorum=127.0.0.1:61024, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T15:56:58,721 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36387-0x1017926911e0003, quorum=127.0.0.1:61024, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T15:56:58,721 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34663-0x1017926911e0000, quorum=127.0.0.1:61024, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T15:56:58,745 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46585 is added to blk_1073741827_1003 (size=196) 2024-12-03T15:56:58,746 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40575 is added to blk_1073741827_1003 (size=196) 2024-12-03T15:56:58,746 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44827 is added to blk_1073741827_1003 (size=196) 2024-12-03T15:56:58,764 INFO [master/713339a81dd9:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-03T15:56:58,766 INFO [master/713339a81dd9:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-12-03T15:56:58,772 INFO [master/713339a81dd9:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-03T15:56:58,809 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46585 is added to blk_1073741828_1004 (size=1189) 2024-12-03T15:56:58,810 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44827 is added to blk_1073741828_1004 (size=1189) 2024-12-03T15:56:58,810 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40575 is added to blk_1073741828_1004 (size=1189) 2024-12-03T15:56:58,836 INFO [master/713339a81dd9:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:45015/user/jenkins/test-data/c81ce3c5-e5b8-1c2f-869e-120d734cf311/MasterData/data/master/store 2024-12-03T15:56:58,855 WARN [IPC Server handler 0 on default port 45015 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-12-03T15:56:58,855 WARN [IPC Server handler 0 on default port 45015 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-12-03T15:56:58,855 WARN [IPC Server handler 0 on default port 45015 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-12-03T15:56:58,865 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40575 is added to blk_1073741829_1005 (size=34) 2024-12-03T15:56:58,865 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44827 is added to blk_1073741829_1005 (size=34) 2024-12-03T15:56:58,871 INFO [master/713339a81dd9:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-12-03T15:56:58,876 DEBUG [master/713339a81dd9:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T15:56:58,877 DEBUG [master/713339a81dd9:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-03T15:56:58,878 INFO [master/713339a81dd9:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-03T15:56:58,878 DEBUG [master/713339a81dd9:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-03T15:56:58,880 DEBUG [master/713339a81dd9:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-03T15:56:58,880 DEBUG [master/713339a81dd9:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-03T15:56:58,880 INFO [master/713339a81dd9:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-03T15:56:58,882 DEBUG [master/713339a81dd9:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733241418877Disabling compacts and flushes for region at 1733241418877Disabling writes for close at 1733241418880 (+3 ms)Writing region close event to WAL at 1733241418880Closed at 1733241418880 2024-12-03T15:56:58,885 WARN [master/713339a81dd9:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:45015/user/jenkins/test-data/c81ce3c5-e5b8-1c2f-869e-120d734cf311/MasterData/data/master/store/.initializing 2024-12-03T15:56:58,885 DEBUG [master/713339a81dd9:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:45015/user/jenkins/test-data/c81ce3c5-e5b8-1c2f-869e-120d734cf311/MasterData/WALs/713339a81dd9,34663,1733241417250 2024-12-03T15:56:58,896 INFO [master/713339a81dd9:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-03T15:56:58,912 INFO [master/713339a81dd9:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=713339a81dd9%2C34663%2C1733241417250, suffix=, logDir=hdfs://localhost:45015/user/jenkins/test-data/c81ce3c5-e5b8-1c2f-869e-120d734cf311/MasterData/WALs/713339a81dd9,34663,1733241417250, archiveDir=hdfs://localhost:45015/user/jenkins/test-data/c81ce3c5-e5b8-1c2f-869e-120d734cf311/MasterData/oldWALs, maxLogs=10 2024-12-03T15:56:58,944 DEBUG [master/713339a81dd9:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/c81ce3c5-e5b8-1c2f-869e-120d734cf311/MasterData/WALs/713339a81dd9,34663,1733241417250/713339a81dd9%2C34663%2C1733241417250.1733241418917, exclude list is [], retry=0 2024-12-03T15:56:58,965 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.initialize(FanOutOneBlockAsyncDFSOutputHelper.java:413) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$5.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:472) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$5.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:467) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.NettyFutureUtils.lambda$addListener$0(NettyFutureUtils.java:56) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setSuccess0(DefaultPromise.java:625) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.trySuccess(DefaultPromise.java:105) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPromise.trySuccess(DefaultChannelPromise.java:84) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:658) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:696) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T15:56:58,971 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40575,DS-309fe9b7-1997-44cb-b0c9-1ce3c04a71de,DISK] 2024-12-03T15:56:58,972 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46585,DS-5c6771fd-75c7-4775-ad88-ae5bbe88de5c,DISK] 2024-12-03T15:56:58,972 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44827,DS-e007f1cc-fc1b-4368-9716-e11e4fe2f4eb,DISK] 2024-12-03T15:56:58,976 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-12-03T15:56:59,018 DEBUG [master/713339a81dd9:0:becomeActiveMaster {}] crypto.Encryption(565): Installed org.apache.hadoop.hbase.io.crypto.MockAesKeyProvider into key provider cache 2024-12-03T15:56:59,104 INFO [master/713339a81dd9:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/c81ce3c5-e5b8-1c2f-869e-120d734cf311/MasterData/WALs/713339a81dd9,34663,1733241417250/713339a81dd9%2C34663%2C1733241417250.1733241418917 2024-12-03T15:56:59,106 DEBUG [master/713339a81dd9:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37355:37355),(127.0.0.1/127.0.0.1:44513:44513),(127.0.0.1/127.0.0.1:46731:46731)] 2024-12-03T15:56:59,107 DEBUG [master/713339a81dd9:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-12-03T15:56:59,107 DEBUG [master/713339a81dd9:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T15:56:59,112 DEBUG [master/713339a81dd9:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-12-03T15:56:59,113 DEBUG [master/713339a81dd9:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-12-03T15:56:59,167 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-12-03T15:56:59,197 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-12-03T15:56:59,200 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T15:56:59,204 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-03T15:56:59,205 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-12-03T15:56:59,209 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-12-03T15:56:59,209 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T15:56:59,210 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T15:56:59,211 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-12-03T15:56:59,214 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-12-03T15:56:59,214 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T15:56:59,215 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T15:56:59,215 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-12-03T15:56:59,218 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-12-03T15:56:59,218 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T15:56:59,219 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T15:56:59,220 DEBUG [master/713339a81dd9:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-12-03T15:56:59,224 DEBUG [master/713339a81dd9:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45015/user/jenkins/test-data/c81ce3c5-e5b8-1c2f-869e-120d734cf311/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-03T15:56:59,226 DEBUG [master/713339a81dd9:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45015/user/jenkins/test-data/c81ce3c5-e5b8-1c2f-869e-120d734cf311/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-03T15:56:59,233 DEBUG [master/713339a81dd9:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-12-03T15:56:59,234 DEBUG [master/713339a81dd9:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-12-03T15:56:59,239 DEBUG [master/713339a81dd9:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-03T15:56:59,244 DEBUG [master/713339a81dd9:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-12-03T15:56:59,250 DEBUG [master/713339a81dd9:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45015/user/jenkins/test-data/c81ce3c5-e5b8-1c2f-869e-120d734cf311/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T15:56:59,252 INFO [master/713339a81dd9:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63174473, jitterRate=-0.058626994490623474}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-03T15:56:59,260 DEBUG [master/713339a81dd9:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1733241419134Initializing all the Stores at 1733241419137 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733241419138 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733241419139 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733241419139Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733241419139Cleaning up temporary data from old regions at 1733241419234 (+95 ms)Region opened successfully at 1733241419260 (+26 ms) 2024-12-03T15:56:59,262 INFO [master/713339a81dd9:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-12-03T15:56:59,302 DEBUG [master/713339a81dd9:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@28f70835, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=713339a81dd9/172.17.0.3:0 2024-12-03T15:56:59,335 INFO [master/713339a81dd9:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-12-03T15:56:59,347 INFO [master/713339a81dd9:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-12-03T15:56:59,348 INFO [master/713339a81dd9:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-12-03T15:56:59,350 INFO [master/713339a81dd9:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-03T15:56:59,352 INFO [master/713339a81dd9:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-12-03T15:56:59,357 INFO [master/713339a81dd9:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-12-03T15:56:59,357 INFO [master/713339a81dd9:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-12-03T15:56:59,385 INFO [master/713339a81dd9:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-12-03T15:56:59,394 DEBUG [master/713339a81dd9:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34663-0x1017926911e0000, quorum=127.0.0.1:61024, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-12-03T15:56:59,396 DEBUG [master/713339a81dd9:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-12-03T15:56:59,399 INFO [master/713339a81dd9:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-12-03T15:56:59,400 DEBUG [master/713339a81dd9:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34663-0x1017926911e0000, quorum=127.0.0.1:61024, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-12-03T15:56:59,402 DEBUG [master/713339a81dd9:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-12-03T15:56:59,404 INFO [master/713339a81dd9:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-12-03T15:56:59,408 DEBUG [master/713339a81dd9:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34663-0x1017926911e0000, quorum=127.0.0.1:61024, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-12-03T15:56:59,410 DEBUG [master/713339a81dd9:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-12-03T15:56:59,411 DEBUG [master/713339a81dd9:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34663-0x1017926911e0000, quorum=127.0.0.1:61024, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-12-03T15:56:59,412 DEBUG [master/713339a81dd9:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-12-03T15:56:59,434 DEBUG [master/713339a81dd9:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34663-0x1017926911e0000, quorum=127.0.0.1:61024, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-12-03T15:56:59,436 DEBUG [master/713339a81dd9:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-12-03T15:56:59,440 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42147-0x1017926911e0001, quorum=127.0.0.1:61024, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-03T15:56:59,440 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34819-0x1017926911e0002, quorum=127.0.0.1:61024, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-03T15:56:59,441 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34663-0x1017926911e0000, quorum=127.0.0.1:61024, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-03T15:56:59,441 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42147-0x1017926911e0001, quorum=127.0.0.1:61024, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T15:56:59,441 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34819-0x1017926911e0002, quorum=127.0.0.1:61024, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T15:56:59,441 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36387-0x1017926911e0003, quorum=127.0.0.1:61024, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-03T15:56:59,441 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36387-0x1017926911e0003, quorum=127.0.0.1:61024, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T15:56:59,443 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34663-0x1017926911e0000, quorum=127.0.0.1:61024, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T15:56:59,446 INFO [master/713339a81dd9:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=713339a81dd9,34663,1733241417250, sessionid=0x1017926911e0000, setting cluster-up flag (Was=false) 2024-12-03T15:56:59,462 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36387-0x1017926911e0003, quorum=127.0.0.1:61024, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T15:56:59,462 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42147-0x1017926911e0001, quorum=127.0.0.1:61024, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T15:56:59,462 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34663-0x1017926911e0000, quorum=127.0.0.1:61024, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T15:56:59,462 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34819-0x1017926911e0002, quorum=127.0.0.1:61024, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T15:56:59,469 DEBUG [master/713339a81dd9:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-12-03T15:56:59,471 DEBUG [master/713339a81dd9:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=713339a81dd9,34663,1733241417250 2024-12-03T15:56:59,476 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34819-0x1017926911e0002, quorum=127.0.0.1:61024, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T15:56:59,477 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42147-0x1017926911e0001, quorum=127.0.0.1:61024, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T15:56:59,477 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36387-0x1017926911e0003, quorum=127.0.0.1:61024, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T15:56:59,477 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34663-0x1017926911e0000, quorum=127.0.0.1:61024, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T15:56:59,484 DEBUG [master/713339a81dd9:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-12-03T15:56:59,485 DEBUG [master/713339a81dd9:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=713339a81dd9,34663,1733241417250 2024-12-03T15:56:59,492 INFO [master/713339a81dd9:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:45015/user/jenkins/test-data/c81ce3c5-e5b8-1c2f-869e-120d734cf311/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-12-03T15:56:59,540 INFO [RS:0;713339a81dd9:42147 {}] regionserver.HRegionServer(746): ClusterId : a4fd42ae-14ff-4383-be19-b09963b80309 2024-12-03T15:56:59,540 INFO [RS:2;713339a81dd9:36387 {}] regionserver.HRegionServer(746): ClusterId : a4fd42ae-14ff-4383-be19-b09963b80309 2024-12-03T15:56:59,543 DEBUG [RS:0;713339a81dd9:42147 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-03T15:56:59,543 INFO [RS:1;713339a81dd9:34819 {}] regionserver.HRegionServer(746): ClusterId : a4fd42ae-14ff-4383-be19-b09963b80309 2024-12-03T15:56:59,544 DEBUG [RS:1;713339a81dd9:34819 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-03T15:56:59,544 DEBUG [RS:2;713339a81dd9:36387 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-03T15:56:59,552 DEBUG [RS:1;713339a81dd9:34819 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-03T15:56:59,552 DEBUG [RS:0;713339a81dd9:42147 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-03T15:56:59,552 DEBUG [RS:2;713339a81dd9:36387 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-03T15:56:59,552 DEBUG [RS:1;713339a81dd9:34819 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-03T15:56:59,552 DEBUG [RS:0;713339a81dd9:42147 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-03T15:56:59,552 DEBUG [RS:2;713339a81dd9:36387 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-03T15:56:59,556 DEBUG [RS:0;713339a81dd9:42147 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-03T15:56:59,556 DEBUG [RS:1;713339a81dd9:34819 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-03T15:56:59,556 DEBUG [RS:2;713339a81dd9:36387 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-03T15:56:59,557 DEBUG [RS:0;713339a81dd9:42147 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@619e7583, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=713339a81dd9/172.17.0.3:0 2024-12-03T15:56:59,557 DEBUG [RS:1;713339a81dd9:34819 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@109447f5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=713339a81dd9/172.17.0.3:0 2024-12-03T15:56:59,566 DEBUG [RS:2;713339a81dd9:36387 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@36a12904, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=713339a81dd9/172.17.0.3:0 2024-12-03T15:56:59,579 DEBUG [RS:0;713339a81dd9:42147 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;713339a81dd9:42147 2024-12-03T15:56:59,580 DEBUG [RS:1;713339a81dd9:34819 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;713339a81dd9:34819 2024-12-03T15:56:59,582 INFO [RS:0;713339a81dd9:42147 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-03T15:56:59,582 INFO [RS:1;713339a81dd9:34819 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-03T15:56:59,582 INFO [RS:0;713339a81dd9:42147 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-03T15:56:59,582 INFO [RS:1;713339a81dd9:34819 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-03T15:56:59,582 DEBUG [RS:0;713339a81dd9:42147 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-03T15:56:59,582 DEBUG [RS:1;713339a81dd9:34819 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-03T15:56:59,585 INFO [RS:0;713339a81dd9:42147 {}] regionserver.HRegionServer(2659): reportForDuty to master=713339a81dd9,34663,1733241417250 with port=42147, startcode=1733241418180 2024-12-03T15:56:59,585 INFO [RS:1;713339a81dd9:34819 {}] regionserver.HRegionServer(2659): reportForDuty to master=713339a81dd9,34663,1733241417250 with port=34819, startcode=1733241418314 2024-12-03T15:56:59,589 DEBUG [master/713339a81dd9:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-03T15:56:59,593 DEBUG [RS:2;713339a81dd9:36387 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:2;713339a81dd9:36387 2024-12-03T15:56:59,594 INFO [RS:2;713339a81dd9:36387 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-03T15:56:59,594 INFO [RS:2;713339a81dd9:36387 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-03T15:56:59,594 DEBUG [RS:2;713339a81dd9:36387 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-03T15:56:59,596 INFO [RS:2;713339a81dd9:36387 {}] regionserver.HRegionServer(2659): reportForDuty to master=713339a81dd9,34663,1733241417250 with port=36387, startcode=1733241418383 2024-12-03T15:56:59,602 DEBUG [RS:0;713339a81dd9:42147 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-03T15:56:59,602 DEBUG [RS:1;713339a81dd9:34819 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-03T15:56:59,602 INFO [master/713339a81dd9:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-12-03T15:56:59,602 DEBUG [RS:2;713339a81dd9:36387 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-03T15:56:59,612 INFO [master/713339a81dd9:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-12-03T15:56:59,619 DEBUG [master/713339a81dd9:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 713339a81dd9,34663,1733241417250 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-12-03T15:56:59,629 DEBUG [master/713339a81dd9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/713339a81dd9:0, corePoolSize=5, maxPoolSize=5 2024-12-03T15:56:59,629 DEBUG [master/713339a81dd9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/713339a81dd9:0, corePoolSize=5, maxPoolSize=5 2024-12-03T15:56:59,629 DEBUG [master/713339a81dd9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/713339a81dd9:0, corePoolSize=5, maxPoolSize=5 2024-12-03T15:56:59,629 DEBUG [master/713339a81dd9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/713339a81dd9:0, corePoolSize=5, maxPoolSize=5 2024-12-03T15:56:59,630 DEBUG [master/713339a81dd9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/713339a81dd9:0, corePoolSize=10, maxPoolSize=10 2024-12-03T15:56:59,630 DEBUG [master/713339a81dd9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/713339a81dd9:0, corePoolSize=1, maxPoolSize=1 2024-12-03T15:56:59,630 DEBUG [master/713339a81dd9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/713339a81dd9:0, corePoolSize=2, maxPoolSize=2 2024-12-03T15:56:59,630 DEBUG [master/713339a81dd9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/713339a81dd9:0, corePoolSize=1, maxPoolSize=1 2024-12-03T15:56:59,664 INFO [HMaster-EventLoopGroup-2-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55125, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.1 (auth:SIMPLE), service=RegionServerStatusService 2024-12-03T15:56:59,664 INFO [HMaster-EventLoopGroup-2-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43365, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.2 (auth:SIMPLE), service=RegionServerStatusService 2024-12-03T15:56:59,671 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34663 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T15:56:59,664 INFO [HMaster-EventLoopGroup-2-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55343, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-12-03T15:56:59,678 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34663 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T15:56:59,679 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34663 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T15:56:59,680 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-03T15:56:59,680 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-12-03T15:56:59,687 INFO [master/713339a81dd9:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733241449687 2024-12-03T15:56:59,689 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T15:56:59,689 INFO [master/713339a81dd9:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-12-03T15:56:59,690 INFO [master/713339a81dd9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-12-03T15:56:59,689 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-12-03T15:56:59,695 INFO [master/713339a81dd9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-12-03T15:56:59,695 INFO [master/713339a81dd9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-12-03T15:56:59,696 INFO [master/713339a81dd9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-12-03T15:56:59,696 INFO [master/713339a81dd9:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-12-03T15:56:59,697 INFO [master/713339a81dd9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-03T15:56:59,714 INFO [master/713339a81dd9:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-12-03T15:56:59,715 DEBUG [RS:0;713339a81dd9:42147 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-12-03T15:56:59,715 DEBUG [RS:2;713339a81dd9:36387 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-12-03T15:56:59,715 WARN [RS:0;713339a81dd9:42147 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-12-03T15:56:59,715 WARN [RS:2;713339a81dd9:36387 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-12-03T15:56:59,716 DEBUG [RS:1;713339a81dd9:34819 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-12-03T15:56:59,716 WARN [RS:1;713339a81dd9:34819 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-12-03T15:56:59,716 INFO [master/713339a81dd9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-12-03T15:56:59,716 INFO [master/713339a81dd9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-12-03T15:56:59,721 INFO [master/713339a81dd9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-12-03T15:56:59,750 INFO [master/713339a81dd9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-12-03T15:56:59,753 DEBUG [master/713339a81dd9:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/713339a81dd9:0:becomeActiveMaster-HFileCleaner.large.0-1733241419752,5,FailOnTimeoutGroup] 2024-12-03T15:56:59,753 DEBUG [master/713339a81dd9:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/713339a81dd9:0:becomeActiveMaster-HFileCleaner.small.0-1733241419753,5,FailOnTimeoutGroup] 2024-12-03T15:56:59,754 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40575 is added to blk_1073741831_1007 (size=1321) 2024-12-03T15:56:59,754 INFO [master/713339a81dd9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-03T15:56:59,754 INFO [master/713339a81dd9:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-12-03T15:56:59,755 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44827 is added to blk_1073741831_1007 (size=1321) 2024-12-03T15:56:59,755 INFO [master/713339a81dd9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-12-03T15:56:59,756 INFO [master/713339a81dd9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-12-03T15:56:59,759 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46585 is added to blk_1073741831_1007 (size=1321) 2024-12-03T15:56:59,760 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:45015/user/jenkins/test-data/c81ce3c5-e5b8-1c2f-869e-120d734cf311/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-12-03T15:56:59,761 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:45015/user/jenkins/test-data/c81ce3c5-e5b8-1c2f-869e-120d734cf311 2024-12-03T15:56:59,777 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46585 is added to blk_1073741832_1008 (size=32) 2024-12-03T15:56:59,778 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44827 is added to blk_1073741832_1008 (size=32) 2024-12-03T15:56:59,778 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40575 is added to blk_1073741832_1008 (size=32) 2024-12-03T15:56:59,788 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T15:56:59,792 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-03T15:56:59,795 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-03T15:56:59,796 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T15:56:59,797 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-03T15:56:59,797 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-03T15:56:59,800 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-03T15:56:59,800 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T15:56:59,802 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-03T15:56:59,802 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-03T15:56:59,805 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-03T15:56:59,805 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T15:56:59,806 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-03T15:56:59,807 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-03T15:56:59,810 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-03T15:56:59,810 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T15:56:59,811 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-03T15:56:59,811 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-03T15:56:59,813 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45015/user/jenkins/test-data/c81ce3c5-e5b8-1c2f-869e-120d734cf311/data/hbase/meta/1588230740 2024-12-03T15:56:59,814 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45015/user/jenkins/test-data/c81ce3c5-e5b8-1c2f-869e-120d734cf311/data/hbase/meta/1588230740 2024-12-03T15:56:59,817 INFO [RS:2;713339a81dd9:36387 {}] regionserver.HRegionServer(2659): reportForDuty to master=713339a81dd9,34663,1733241417250 with port=36387, startcode=1733241418383 2024-12-03T15:56:59,818 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-03T15:56:59,818 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-03T15:56:59,818 INFO [RS:1;713339a81dd9:34819 {}] regionserver.HRegionServer(2659): reportForDuty to master=713339a81dd9,34663,1733241417250 with port=34819, startcode=1733241418314 2024-12-03T15:56:59,818 INFO [RS:0;713339a81dd9:42147 {}] regionserver.HRegionServer(2659): reportForDuty to master=713339a81dd9,34663,1733241417250 with port=42147, startcode=1733241418180 2024-12-03T15:56:59,819 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-03T15:56:59,819 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34663 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 713339a81dd9,36387,1733241418383 2024-12-03T15:56:59,822 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-03T15:56:59,822 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34663 {}] master.ServerManager(517): Registering regionserver=713339a81dd9,36387,1733241418383 2024-12-03T15:56:59,826 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45015/user/jenkins/test-data/c81ce3c5-e5b8-1c2f-869e-120d734cf311/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T15:56:59,828 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61433355, jitterRate=-0.08457167446613312}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-03T15:56:59,831 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1733241419789Initializing all the Stores at 1733241419791 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733241419791Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733241419791Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733241419791Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733241419791Cleaning up temporary data from old regions at 1733241419818 (+27 ms)Region opened successfully at 1733241419831 (+13 ms) 2024-12-03T15:56:59,831 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-03T15:56:59,831 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-03T15:56:59,831 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-03T15:56:59,832 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-03T15:56:59,832 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-03T15:56:59,835 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34663 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 713339a81dd9,42147,1733241418180 2024-12-03T15:56:59,835 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34663 {}] master.ServerManager(517): Registering regionserver=713339a81dd9,42147,1733241418180 2024-12-03T15:56:59,837 DEBUG [RS:2;713339a81dd9:36387 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:45015/user/jenkins/test-data/c81ce3c5-e5b8-1c2f-869e-120d734cf311 2024-12-03T15:56:59,837 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-03T15:56:59,837 DEBUG [RS:2;713339a81dd9:36387 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:45015 2024-12-03T15:56:59,837 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733241419831Disabling compacts and flushes for region at 1733241419831Disabling writes for close at 1733241419832 (+1 ms)Writing region close event to WAL at 1733241419836 (+4 ms)Closed at 1733241419837 (+1 ms) 2024-12-03T15:56:59,837 DEBUG [RS:2;713339a81dd9:36387 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-03T15:56:59,841 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-03T15:56:59,841 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-12-03T15:56:59,841 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34663 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 713339a81dd9,34819,1733241418314 2024-12-03T15:56:59,842 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34663 {}] master.ServerManager(517): Registering regionserver=713339a81dd9,34819,1733241418314 2024-12-03T15:56:59,842 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34663-0x1017926911e0000, quorum=127.0.0.1:61024, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-03T15:56:59,843 DEBUG [RS:2;713339a81dd9:36387 {}] zookeeper.ZKUtil(111): regionserver:36387-0x1017926911e0003, quorum=127.0.0.1:61024, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/713339a81dd9,36387,1733241418383 2024-12-03T15:56:59,843 WARN [RS:2;713339a81dd9:36387 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-03T15:56:59,844 INFO [RS:2;713339a81dd9:36387 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-03T15:56:59,844 DEBUG [RS:2;713339a81dd9:36387 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:45015/user/jenkins/test-data/c81ce3c5-e5b8-1c2f-869e-120d734cf311/WALs/713339a81dd9,36387,1733241418383 2024-12-03T15:56:59,846 DEBUG [RS:0;713339a81dd9:42147 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:45015/user/jenkins/test-data/c81ce3c5-e5b8-1c2f-869e-120d734cf311 2024-12-03T15:56:59,846 DEBUG [RS:0;713339a81dd9:42147 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:45015 2024-12-03T15:56:59,846 DEBUG [RS:0;713339a81dd9:42147 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-03T15:56:59,847 DEBUG [RS:1;713339a81dd9:34819 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:45015/user/jenkins/test-data/c81ce3c5-e5b8-1c2f-869e-120d734cf311 2024-12-03T15:56:59,847 DEBUG [RS:1;713339a81dd9:34819 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:45015 2024-12-03T15:56:59,847 DEBUG [RS:1;713339a81dd9:34819 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-03T15:56:59,848 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [713339a81dd9,36387,1733241418383] 2024-12-03T15:56:59,850 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-12-03T15:56:59,853 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34663-0x1017926911e0000, quorum=127.0.0.1:61024, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-03T15:56:59,854 DEBUG [RS:0;713339a81dd9:42147 {}] zookeeper.ZKUtil(111): regionserver:42147-0x1017926911e0001, quorum=127.0.0.1:61024, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/713339a81dd9,42147,1733241418180 2024-12-03T15:56:59,854 WARN [RS:0;713339a81dd9:42147 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-03T15:56:59,854 INFO [RS:0;713339a81dd9:42147 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-03T15:56:59,855 DEBUG [RS:1;713339a81dd9:34819 {}] zookeeper.ZKUtil(111): regionserver:34819-0x1017926911e0002, quorum=127.0.0.1:61024, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/713339a81dd9,34819,1733241418314 2024-12-03T15:56:59,858 WARN [RS:1;713339a81dd9:34819 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-03T15:56:59,858 INFO [RS:1;713339a81dd9:34819 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-03T15:56:59,859 DEBUG [RS:1;713339a81dd9:34819 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:45015/user/jenkins/test-data/c81ce3c5-e5b8-1c2f-869e-120d734cf311/WALs/713339a81dd9,34819,1733241418314 2024-12-03T15:56:59,860 DEBUG [RS:0;713339a81dd9:42147 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:45015/user/jenkins/test-data/c81ce3c5-e5b8-1c2f-869e-120d734cf311/WALs/713339a81dd9,42147,1733241418180 2024-12-03T15:56:59,862 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [713339a81dd9,42147,1733241418180] 2024-12-03T15:56:59,862 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [713339a81dd9,34819,1733241418314] 2024-12-03T15:56:59,867 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-03T15:56:59,872 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-12-03T15:56:59,886 INFO [RS:0;713339a81dd9:42147 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-03T15:56:59,886 INFO [RS:2;713339a81dd9:36387 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-03T15:56:59,886 INFO [RS:1;713339a81dd9:34819 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-03T15:56:59,903 INFO [RS:0;713339a81dd9:42147 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-03T15:56:59,903 INFO [RS:2;713339a81dd9:36387 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-03T15:56:59,905 INFO [RS:1;713339a81dd9:34819 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-03T15:56:59,910 INFO [RS:0;713339a81dd9:42147 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-03T15:56:59,910 INFO [RS:1;713339a81dd9:34819 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-03T15:56:59,910 INFO [RS:2;713339a81dd9:36387 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-03T15:56:59,911 INFO [RS:2;713339a81dd9:36387 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-03T15:56:59,911 INFO [RS:1;713339a81dd9:34819 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-03T15:56:59,911 INFO [RS:0;713339a81dd9:42147 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-03T15:56:59,919 INFO [RS:1;713339a81dd9:34819 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-03T15:56:59,919 INFO [RS:2;713339a81dd9:36387 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-03T15:56:59,919 INFO [RS:0;713339a81dd9:42147 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-03T15:56:59,926 INFO [RS:0;713339a81dd9:42147 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-03T15:56:59,926 INFO [RS:1;713339a81dd9:34819 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-03T15:56:59,927 INFO [RS:2;713339a81dd9:36387 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-03T15:56:59,928 INFO [RS:2;713339a81dd9:36387 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-03T15:56:59,929 DEBUG [RS:2;713339a81dd9:36387 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/713339a81dd9:0, corePoolSize=1, maxPoolSize=1 2024-12-03T15:56:59,929 DEBUG [RS:2;713339a81dd9:36387 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/713339a81dd9:0, corePoolSize=1, maxPoolSize=1 2024-12-03T15:56:59,929 DEBUG [RS:2;713339a81dd9:36387 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/713339a81dd9:0, corePoolSize=1, maxPoolSize=1 2024-12-03T15:56:59,929 DEBUG [RS:2;713339a81dd9:36387 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/713339a81dd9:0, corePoolSize=1, maxPoolSize=1 2024-12-03T15:56:59,930 DEBUG [RS:2;713339a81dd9:36387 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/713339a81dd9:0, corePoolSize=1, maxPoolSize=1 2024-12-03T15:56:59,930 INFO [RS:0;713339a81dd9:42147 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-03T15:56:59,930 DEBUG [RS:2;713339a81dd9:36387 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/713339a81dd9:0, corePoolSize=2, maxPoolSize=2 2024-12-03T15:56:59,930 DEBUG [RS:2;713339a81dd9:36387 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/713339a81dd9:0, corePoolSize=1, maxPoolSize=1 2024-12-03T15:56:59,930 DEBUG [RS:0;713339a81dd9:42147 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/713339a81dd9:0, corePoolSize=1, maxPoolSize=1 2024-12-03T15:56:59,930 DEBUG [RS:2;713339a81dd9:36387 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/713339a81dd9:0, corePoolSize=1, maxPoolSize=1 2024-12-03T15:56:59,930 DEBUG [RS:0;713339a81dd9:42147 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/713339a81dd9:0, corePoolSize=1, maxPoolSize=1 2024-12-03T15:56:59,930 DEBUG [RS:2;713339a81dd9:36387 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/713339a81dd9:0, corePoolSize=1, maxPoolSize=1 2024-12-03T15:56:59,931 DEBUG [RS:0;713339a81dd9:42147 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/713339a81dd9:0, corePoolSize=1, maxPoolSize=1 2024-12-03T15:56:59,931 DEBUG [RS:2;713339a81dd9:36387 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/713339a81dd9:0, corePoolSize=1, maxPoolSize=1 2024-12-03T15:56:59,931 DEBUG [RS:0;713339a81dd9:42147 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/713339a81dd9:0, corePoolSize=1, maxPoolSize=1 2024-12-03T15:56:59,931 DEBUG [RS:2;713339a81dd9:36387 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/713339a81dd9:0, corePoolSize=1, maxPoolSize=1 2024-12-03T15:56:59,931 DEBUG [RS:0;713339a81dd9:42147 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/713339a81dd9:0, corePoolSize=1, maxPoolSize=1 2024-12-03T15:56:59,931 DEBUG [RS:2;713339a81dd9:36387 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/713339a81dd9:0, corePoolSize=1, maxPoolSize=1 2024-12-03T15:56:59,931 DEBUG [RS:0;713339a81dd9:42147 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/713339a81dd9:0, corePoolSize=2, maxPoolSize=2 2024-12-03T15:56:59,931 DEBUG [RS:2;713339a81dd9:36387 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/713339a81dd9:0, corePoolSize=3, maxPoolSize=3 2024-12-03T15:56:59,931 DEBUG [RS:0;713339a81dd9:42147 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/713339a81dd9:0, corePoolSize=1, maxPoolSize=1 2024-12-03T15:56:59,932 DEBUG [RS:2;713339a81dd9:36387 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/713339a81dd9:0, corePoolSize=3, maxPoolSize=3 2024-12-03T15:56:59,932 DEBUG [RS:0;713339a81dd9:42147 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/713339a81dd9:0, corePoolSize=1, maxPoolSize=1 2024-12-03T15:56:59,932 DEBUG [RS:0;713339a81dd9:42147 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/713339a81dd9:0, corePoolSize=1, maxPoolSize=1 2024-12-03T15:56:59,932 INFO [RS:1;713339a81dd9:34819 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-03T15:56:59,932 DEBUG [RS:0;713339a81dd9:42147 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/713339a81dd9:0, corePoolSize=1, maxPoolSize=1 2024-12-03T15:56:59,932 DEBUG [RS:0;713339a81dd9:42147 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/713339a81dd9:0, corePoolSize=1, maxPoolSize=1 2024-12-03T15:56:59,932 DEBUG [RS:1;713339a81dd9:34819 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/713339a81dd9:0, corePoolSize=1, maxPoolSize=1 2024-12-03T15:56:59,932 DEBUG [RS:0;713339a81dd9:42147 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/713339a81dd9:0, corePoolSize=1, maxPoolSize=1 2024-12-03T15:56:59,932 DEBUG [RS:1;713339a81dd9:34819 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/713339a81dd9:0, corePoolSize=1, maxPoolSize=1 2024-12-03T15:56:59,932 DEBUG [RS:0;713339a81dd9:42147 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/713339a81dd9:0, corePoolSize=3, maxPoolSize=3 2024-12-03T15:56:59,933 DEBUG [RS:1;713339a81dd9:34819 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/713339a81dd9:0, corePoolSize=1, maxPoolSize=1 2024-12-03T15:56:59,933 DEBUG [RS:0;713339a81dd9:42147 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/713339a81dd9:0, corePoolSize=3, maxPoolSize=3 2024-12-03T15:56:59,933 DEBUG [RS:1;713339a81dd9:34819 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/713339a81dd9:0, corePoolSize=1, maxPoolSize=1 2024-12-03T15:56:59,933 DEBUG [RS:1;713339a81dd9:34819 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/713339a81dd9:0, corePoolSize=1, maxPoolSize=1 2024-12-03T15:56:59,933 DEBUG [RS:1;713339a81dd9:34819 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/713339a81dd9:0, corePoolSize=2, maxPoolSize=2 2024-12-03T15:56:59,933 DEBUG [RS:1;713339a81dd9:34819 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/713339a81dd9:0, corePoolSize=1, maxPoolSize=1 2024-12-03T15:56:59,934 DEBUG [RS:1;713339a81dd9:34819 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/713339a81dd9:0, corePoolSize=1, maxPoolSize=1 2024-12-03T15:56:59,934 DEBUG [RS:1;713339a81dd9:34819 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/713339a81dd9:0, corePoolSize=1, maxPoolSize=1 2024-12-03T15:56:59,934 DEBUG [RS:1;713339a81dd9:34819 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/713339a81dd9:0, corePoolSize=1, maxPoolSize=1 2024-12-03T15:56:59,934 DEBUG [RS:1;713339a81dd9:34819 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/713339a81dd9:0, corePoolSize=1, maxPoolSize=1 2024-12-03T15:56:59,934 DEBUG [RS:1;713339a81dd9:34819 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/713339a81dd9:0, corePoolSize=1, maxPoolSize=1 2024-12-03T15:56:59,935 DEBUG [RS:1;713339a81dd9:34819 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/713339a81dd9:0, corePoolSize=3, maxPoolSize=3 2024-12-03T15:56:59,935 DEBUG [RS:1;713339a81dd9:34819 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/713339a81dd9:0, corePoolSize=3, maxPoolSize=3 2024-12-03T15:56:59,938 INFO [RS:2;713339a81dd9:36387 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-03T15:56:59,938 INFO [RS:2;713339a81dd9:36387 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-03T15:56:59,938 INFO [RS:2;713339a81dd9:36387 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-03T15:56:59,938 INFO [RS:2;713339a81dd9:36387 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-03T15:56:59,939 INFO [RS:2;713339a81dd9:36387 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-03T15:56:59,939 INFO [RS:2;713339a81dd9:36387 {}] hbase.ChoreService(168): Chore ScheduledChore name=713339a81dd9,36387,1733241418383-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-03T15:56:59,954 INFO [RS:0;713339a81dd9:42147 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-03T15:56:59,954 INFO [RS:0;713339a81dd9:42147 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-03T15:56:59,954 INFO [RS:0;713339a81dd9:42147 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-03T15:56:59,954 INFO [RS:0;713339a81dd9:42147 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-03T15:56:59,954 INFO [RS:0;713339a81dd9:42147 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-03T15:56:59,955 INFO [RS:0;713339a81dd9:42147 {}] hbase.ChoreService(168): Chore ScheduledChore name=713339a81dd9,42147,1733241418180-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-03T15:56:59,962 INFO [RS:1;713339a81dd9:34819 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-03T15:56:59,962 INFO [RS:1;713339a81dd9:34819 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-03T15:56:59,962 INFO [RS:1;713339a81dd9:34819 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-03T15:56:59,962 INFO [RS:1;713339a81dd9:34819 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-03T15:56:59,962 INFO [RS:1;713339a81dd9:34819 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-03T15:56:59,963 INFO [RS:1;713339a81dd9:34819 {}] hbase.ChoreService(168): Chore ScheduledChore name=713339a81dd9,34819,1733241418314-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-03T15:56:59,976 INFO [RS:2;713339a81dd9:36387 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-03T15:56:59,979 INFO [RS:2;713339a81dd9:36387 {}] hbase.ChoreService(168): Chore ScheduledChore name=713339a81dd9,36387,1733241418383-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-03T15:56:59,979 INFO [RS:2;713339a81dd9:36387 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-03T15:56:59,980 INFO [RS:2;713339a81dd9:36387 {}] regionserver.Replication(171): 713339a81dd9,36387,1733241418383 started 2024-12-03T15:56:59,983 INFO [RS:0;713339a81dd9:42147 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-03T15:56:59,983 INFO [RS:0;713339a81dd9:42147 {}] hbase.ChoreService(168): Chore ScheduledChore name=713339a81dd9,42147,1733241418180-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-03T15:56:59,983 INFO [RS:0;713339a81dd9:42147 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-03T15:56:59,983 INFO [RS:0;713339a81dd9:42147 {}] regionserver.Replication(171): 713339a81dd9,42147,1733241418180 started 2024-12-03T15:56:59,993 INFO [RS:1;713339a81dd9:34819 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-03T15:56:59,994 INFO [RS:1;713339a81dd9:34819 {}] hbase.ChoreService(168): Chore ScheduledChore name=713339a81dd9,34819,1733241418314-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-03T15:56:59,994 INFO [RS:1;713339a81dd9:34819 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-03T15:56:59,994 INFO [RS:1;713339a81dd9:34819 {}] regionserver.Replication(171): 713339a81dd9,34819,1733241418314 started 2024-12-03T15:57:00,009 INFO [RS:0;713339a81dd9:42147 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-03T15:57:00,009 INFO [RS:0;713339a81dd9:42147 {}] regionserver.HRegionServer(1482): Serving as 713339a81dd9,42147,1733241418180, RpcServer on 713339a81dd9/172.17.0.3:42147, sessionid=0x1017926911e0001 2024-12-03T15:57:00,010 INFO [RS:2;713339a81dd9:36387 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-03T15:57:00,010 INFO [RS:2;713339a81dd9:36387 {}] regionserver.HRegionServer(1482): Serving as 713339a81dd9,36387,1733241418383, RpcServer on 713339a81dd9/172.17.0.3:36387, sessionid=0x1017926911e0003 2024-12-03T15:57:00,010 DEBUG [RS:0;713339a81dd9:42147 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-03T15:57:00,010 DEBUG [RS:2;713339a81dd9:36387 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-03T15:57:00,010 DEBUG [RS:2;713339a81dd9:36387 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 713339a81dd9,36387,1733241418383 2024-12-03T15:57:00,010 DEBUG [RS:0;713339a81dd9:42147 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 713339a81dd9,42147,1733241418180 2024-12-03T15:57:00,011 DEBUG [RS:2;713339a81dd9:36387 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '713339a81dd9,36387,1733241418383' 2024-12-03T15:57:00,011 DEBUG [RS:0;713339a81dd9:42147 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '713339a81dd9,42147,1733241418180' 2024-12-03T15:57:00,011 DEBUG [RS:2;713339a81dd9:36387 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-03T15:57:00,011 DEBUG [RS:0;713339a81dd9:42147 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-03T15:57:00,012 DEBUG [RS:2;713339a81dd9:36387 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-03T15:57:00,012 DEBUG [RS:0;713339a81dd9:42147 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-03T15:57:00,013 DEBUG [RS:2;713339a81dd9:36387 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-03T15:57:00,013 DEBUG [RS:0;713339a81dd9:42147 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-03T15:57:00,013 DEBUG [RS:2;713339a81dd9:36387 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-03T15:57:00,013 DEBUG [RS:0;713339a81dd9:42147 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-03T15:57:00,013 DEBUG [RS:0;713339a81dd9:42147 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 713339a81dd9,42147,1733241418180 2024-12-03T15:57:00,013 DEBUG [RS:2;713339a81dd9:36387 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 713339a81dd9,36387,1733241418383 2024-12-03T15:57:00,013 DEBUG [RS:2;713339a81dd9:36387 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '713339a81dd9,36387,1733241418383' 2024-12-03T15:57:00,013 DEBUG [RS:2;713339a81dd9:36387 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-03T15:57:00,013 DEBUG [RS:0;713339a81dd9:42147 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '713339a81dd9,42147,1733241418180' 2024-12-03T15:57:00,013 DEBUG [RS:0;713339a81dd9:42147 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-03T15:57:00,014 DEBUG [RS:2;713339a81dd9:36387 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-03T15:57:00,014 DEBUG [RS:0;713339a81dd9:42147 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-03T15:57:00,015 DEBUG [RS:2;713339a81dd9:36387 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-03T15:57:00,015 DEBUG [RS:0;713339a81dd9:42147 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-03T15:57:00,015 INFO [RS:0;713339a81dd9:42147 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-03T15:57:00,015 INFO [RS:2;713339a81dd9:36387 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-03T15:57:00,015 INFO [RS:0;713339a81dd9:42147 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-03T15:57:00,015 INFO [RS:2;713339a81dd9:36387 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-03T15:57:00,021 INFO [RS:1;713339a81dd9:34819 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-03T15:57:00,022 INFO [RS:1;713339a81dd9:34819 {}] regionserver.HRegionServer(1482): Serving as 713339a81dd9,34819,1733241418314, RpcServer on 713339a81dd9/172.17.0.3:34819, sessionid=0x1017926911e0002 2024-12-03T15:57:00,022 DEBUG [RS:1;713339a81dd9:34819 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-03T15:57:00,022 DEBUG [RS:1;713339a81dd9:34819 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 713339a81dd9,34819,1733241418314 2024-12-03T15:57:00,022 DEBUG [RS:1;713339a81dd9:34819 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '713339a81dd9,34819,1733241418314' 2024-12-03T15:57:00,022 DEBUG [RS:1;713339a81dd9:34819 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-03T15:57:00,023 WARN [713339a81dd9:34663 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-03T15:57:00,023 DEBUG [RS:1;713339a81dd9:34819 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-03T15:57:00,024 DEBUG [RS:1;713339a81dd9:34819 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-03T15:57:00,024 DEBUG [RS:1;713339a81dd9:34819 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-03T15:57:00,024 DEBUG [RS:1;713339a81dd9:34819 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 713339a81dd9,34819,1733241418314 2024-12-03T15:57:00,024 DEBUG [RS:1;713339a81dd9:34819 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '713339a81dd9,34819,1733241418314' 2024-12-03T15:57:00,024 DEBUG [RS:1;713339a81dd9:34819 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-03T15:57:00,025 DEBUG [RS:1;713339a81dd9:34819 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-03T15:57:00,025 DEBUG [RS:1;713339a81dd9:34819 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-03T15:57:00,025 INFO [RS:1;713339a81dd9:34819 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-03T15:57:00,025 INFO [RS:1;713339a81dd9:34819 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-03T15:57:00,121 INFO [RS:0;713339a81dd9:42147 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-03T15:57:00,122 INFO [RS:2;713339a81dd9:36387 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-03T15:57:00,126 INFO [RS:0;713339a81dd9:42147 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=713339a81dd9%2C42147%2C1733241418180, suffix=, logDir=hdfs://localhost:45015/user/jenkins/test-data/c81ce3c5-e5b8-1c2f-869e-120d734cf311/WALs/713339a81dd9,42147,1733241418180, archiveDir=hdfs://localhost:45015/user/jenkins/test-data/c81ce3c5-e5b8-1c2f-869e-120d734cf311/oldWALs, maxLogs=32 2024-12-03T15:57:00,127 INFO [RS:1;713339a81dd9:34819 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-03T15:57:00,128 INFO [RS:2;713339a81dd9:36387 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=713339a81dd9%2C36387%2C1733241418383, suffix=, logDir=hdfs://localhost:45015/user/jenkins/test-data/c81ce3c5-e5b8-1c2f-869e-120d734cf311/WALs/713339a81dd9,36387,1733241418383, archiveDir=hdfs://localhost:45015/user/jenkins/test-data/c81ce3c5-e5b8-1c2f-869e-120d734cf311/oldWALs, maxLogs=32 2024-12-03T15:57:00,138 INFO [RS:1;713339a81dd9:34819 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=713339a81dd9%2C34819%2C1733241418314, suffix=, logDir=hdfs://localhost:45015/user/jenkins/test-data/c81ce3c5-e5b8-1c2f-869e-120d734cf311/WALs/713339a81dd9,34819,1733241418314, archiveDir=hdfs://localhost:45015/user/jenkins/test-data/c81ce3c5-e5b8-1c2f-869e-120d734cf311/oldWALs, maxLogs=32 2024-12-03T15:57:00,155 DEBUG [RS:2;713339a81dd9:36387 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/c81ce3c5-e5b8-1c2f-869e-120d734cf311/WALs/713339a81dd9,36387,1733241418383/713339a81dd9%2C36387%2C1733241418383.1733241420133, exclude list is [], retry=0 2024-12-03T15:57:00,155 DEBUG [RS:0;713339a81dd9:42147 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/c81ce3c5-e5b8-1c2f-869e-120d734cf311/WALs/713339a81dd9,42147,1733241418180/713339a81dd9%2C42147%2C1733241418180.1733241420132, exclude list is [], retry=0 2024-12-03T15:57:00,170 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44827,DS-e007f1cc-fc1b-4368-9716-e11e4fe2f4eb,DISK] 2024-12-03T15:57:00,170 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40575,DS-309fe9b7-1997-44cb-b0c9-1ce3c04a71de,DISK] 2024-12-03T15:57:00,171 DEBUG [RS:1;713339a81dd9:34819 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/c81ce3c5-e5b8-1c2f-869e-120d734cf311/WALs/713339a81dd9,34819,1733241418314/713339a81dd9%2C34819%2C1733241418314.1733241420140, exclude list is [], retry=0 2024-12-03T15:57:00,172 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46585,DS-5c6771fd-75c7-4775-ad88-ae5bbe88de5c,DISK] 2024-12-03T15:57:00,172 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44827,DS-e007f1cc-fc1b-4368-9716-e11e4fe2f4eb,DISK] 2024-12-03T15:57:00,172 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46585,DS-5c6771fd-75c7-4775-ad88-ae5bbe88de5c,DISK] 2024-12-03T15:57:00,172 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40575,DS-309fe9b7-1997-44cb-b0c9-1ce3c04a71de,DISK] 2024-12-03T15:57:00,182 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46585,DS-5c6771fd-75c7-4775-ad88-ae5bbe88de5c,DISK] 2024-12-03T15:57:00,190 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40575,DS-309fe9b7-1997-44cb-b0c9-1ce3c04a71de,DISK] 2024-12-03T15:57:00,190 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44827,DS-e007f1cc-fc1b-4368-9716-e11e4fe2f4eb,DISK] 2024-12-03T15:57:00,245 INFO [RS:1;713339a81dd9:34819 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/c81ce3c5-e5b8-1c2f-869e-120d734cf311/WALs/713339a81dd9,34819,1733241418314/713339a81dd9%2C34819%2C1733241418314.1733241420140 2024-12-03T15:57:00,246 DEBUG [RS:1;713339a81dd9:34819 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37355:37355),(127.0.0.1/127.0.0.1:46731:46731),(127.0.0.1/127.0.0.1:44513:44513)] 2024-12-03T15:57:00,250 INFO [RS:0;713339a81dd9:42147 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/c81ce3c5-e5b8-1c2f-869e-120d734cf311/WALs/713339a81dd9,42147,1733241418180/713339a81dd9%2C42147%2C1733241418180.1733241420132 2024-12-03T15:57:00,251 DEBUG [RS:0;713339a81dd9:42147 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37355:37355),(127.0.0.1/127.0.0.1:46731:46731),(127.0.0.1/127.0.0.1:44513:44513)] 2024-12-03T15:57:00,252 INFO [RS:2;713339a81dd9:36387 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/c81ce3c5-e5b8-1c2f-869e-120d734cf311/WALs/713339a81dd9,36387,1733241418383/713339a81dd9%2C36387%2C1733241418383.1733241420133 2024-12-03T15:57:00,253 DEBUG [RS:2;713339a81dd9:36387 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44513:44513),(127.0.0.1/127.0.0.1:46731:46731),(127.0.0.1/127.0.0.1:37355:37355)] 2024-12-03T15:57:00,276 DEBUG [713339a81dd9:34663 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=3, allServersCount=3 2024-12-03T15:57:00,287 DEBUG [713339a81dd9:34663 {}] balancer.BalancerClusterState(204): Hosts are {713339a81dd9=0} racks are {/default-rack=0} 2024-12-03T15:57:00,296 DEBUG [713339a81dd9:34663 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-12-03T15:57:00,296 DEBUG [713339a81dd9:34663 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-12-03T15:57:00,296 DEBUG [713339a81dd9:34663 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-12-03T15:57:00,296 DEBUG [713339a81dd9:34663 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-03T15:57:00,296 DEBUG [713339a81dd9:34663 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-03T15:57:00,296 DEBUG [713339a81dd9:34663 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-12-03T15:57:00,296 INFO [713339a81dd9:34663 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-03T15:57:00,296 INFO [713339a81dd9:34663 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-03T15:57:00,296 INFO [713339a81dd9:34663 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-12-03T15:57:00,297 DEBUG [713339a81dd9:34663 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-12-03T15:57:00,306 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=713339a81dd9,34819,1733241418314 2024-12-03T15:57:00,313 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 713339a81dd9,34819,1733241418314, state=OPENING 2024-12-03T15:57:00,320 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-12-03T15:57:00,322 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42147-0x1017926911e0001, quorum=127.0.0.1:61024, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T15:57:00,322 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36387-0x1017926911e0003, quorum=127.0.0.1:61024, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T15:57:00,323 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34663-0x1017926911e0000, quorum=127.0.0.1:61024, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T15:57:00,323 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T15:57:00,324 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T15:57:00,326 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34819-0x1017926911e0002, quorum=127.0.0.1:61024, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T15:57:00,327 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T15:57:00,328 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T15:57:00,328 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-03T15:57:00,331 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=713339a81dd9,34819,1733241418314}] 2024-12-03T15:57:00,522 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-03T15:57:00,526 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34189, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-03T15:57:00,542 INFO [RS_OPEN_META-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-12-03T15:57:00,543 INFO [RS_OPEN_META-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-03T15:57:00,543 INFO [RS_OPEN_META-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-12-03T15:57:00,547 INFO [RS_OPEN_META-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=713339a81dd9%2C34819%2C1733241418314.meta, suffix=.meta, logDir=hdfs://localhost:45015/user/jenkins/test-data/c81ce3c5-e5b8-1c2f-869e-120d734cf311/WALs/713339a81dd9,34819,1733241418314, archiveDir=hdfs://localhost:45015/user/jenkins/test-data/c81ce3c5-e5b8-1c2f-869e-120d734cf311/oldWALs, maxLogs=32 2024-12-03T15:57:00,564 DEBUG [RS_OPEN_META-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/c81ce3c5-e5b8-1c2f-869e-120d734cf311/WALs/713339a81dd9,34819,1733241418314/713339a81dd9%2C34819%2C1733241418314.meta.1733241420549.meta, exclude list is [], retry=0 2024-12-03T15:57:00,568 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46585,DS-5c6771fd-75c7-4775-ad88-ae5bbe88de5c,DISK] 2024-12-03T15:57:00,568 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44827,DS-e007f1cc-fc1b-4368-9716-e11e4fe2f4eb,DISK] 2024-12-03T15:57:00,569 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40575,DS-309fe9b7-1997-44cb-b0c9-1ce3c04a71de,DISK] 2024-12-03T15:57:00,604 INFO [RS_OPEN_META-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/c81ce3c5-e5b8-1c2f-869e-120d734cf311/WALs/713339a81dd9,34819,1733241418314/713339a81dd9%2C34819%2C1733241418314.meta.1733241420549.meta 2024-12-03T15:57:00,604 DEBUG [RS_OPEN_META-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37355:37355),(127.0.0.1/127.0.0.1:46731:46731),(127.0.0.1/127.0.0.1:44513:44513)] 2024-12-03T15:57:00,605 DEBUG [RS_OPEN_META-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-12-03T15:57:00,607 DEBUG [RS_OPEN_META-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-12-03T15:57:00,611 DEBUG [RS_OPEN_META-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-12-03T15:57:00,618 INFO [RS_OPEN_META-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-12-03T15:57:00,624 DEBUG [RS_OPEN_META-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-12-03T15:57:00,625 DEBUG [RS_OPEN_META-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T15:57:00,626 DEBUG [RS_OPEN_META-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-12-03T15:57:00,626 DEBUG [RS_OPEN_META-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-12-03T15:57:00,631 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-03T15:57:00,633 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-03T15:57:00,633 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T15:57:00,634 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-03T15:57:00,635 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-03T15:57:00,636 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-03T15:57:00,636 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T15:57:00,638 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-03T15:57:00,638 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-03T15:57:00,639 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-03T15:57:00,640 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T15:57:00,641 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-03T15:57:00,641 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-03T15:57:00,642 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-03T15:57:00,643 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T15:57:00,643 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-03T15:57:00,644 DEBUG [RS_OPEN_META-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-03T15:57:00,646 DEBUG [RS_OPEN_META-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45015/user/jenkins/test-data/c81ce3c5-e5b8-1c2f-869e-120d734cf311/data/hbase/meta/1588230740 2024-12-03T15:57:00,649 DEBUG [RS_OPEN_META-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45015/user/jenkins/test-data/c81ce3c5-e5b8-1c2f-869e-120d734cf311/data/hbase/meta/1588230740 2024-12-03T15:57:00,653 DEBUG [RS_OPEN_META-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-03T15:57:00,653 DEBUG [RS_OPEN_META-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-03T15:57:00,654 DEBUG [RS_OPEN_META-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-03T15:57:00,657 DEBUG [RS_OPEN_META-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-03T15:57:00,659 INFO [RS_OPEN_META-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67353624, jitterRate=0.0036472082138061523}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-03T15:57:00,659 DEBUG [RS_OPEN_META-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-12-03T15:57:00,661 DEBUG [RS_OPEN_META-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1733241420627Writing region info on filesystem at 1733241420627Initializing all the Stores at 1733241420630 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733241420630Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733241420630Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733241420630Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733241420630Cleaning up temporary data from old regions at 1733241420653 (+23 ms)Running coprocessor post-open hooks at 1733241420659 (+6 ms)Region opened successfully at 1733241420661 (+2 ms) 2024-12-03T15:57:00,675 INFO [RS_OPEN_META-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733241420510 2024-12-03T15:57:00,690 DEBUG [RS_OPEN_META-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-12-03T15:57:00,691 INFO [RS_OPEN_META-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-12-03T15:57:00,694 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=713339a81dd9,34819,1733241418314 2024-12-03T15:57:00,697 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 713339a81dd9,34819,1733241418314, state=OPEN 2024-12-03T15:57:00,700 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34819-0x1017926911e0002, quorum=127.0.0.1:61024, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-03T15:57:00,700 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36387-0x1017926911e0003, quorum=127.0.0.1:61024, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-03T15:57:00,700 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42147-0x1017926911e0001, quorum=127.0.0.1:61024, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-03T15:57:00,700 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T15:57:00,701 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T15:57:00,701 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T15:57:00,701 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34663-0x1017926911e0000, quorum=127.0.0.1:61024, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-03T15:57:00,701 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T15:57:00,702 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=713339a81dd9,34819,1733241418314 2024-12-03T15:57:00,710 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-12-03T15:57:00,711 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=713339a81dd9,34819,1733241418314 in 371 msec 2024-12-03T15:57:00,720 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-12-03T15:57:00,720 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 863 msec 2024-12-03T15:57:00,723 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-03T15:57:00,723 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-12-03T15:57:00,747 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T15:57:00,749 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=713339a81dd9,34819,1733241418314, seqNum=-1] 2024-12-03T15:57:00,774 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T15:57:00,777 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49219, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T15:57:00,816 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.2880 sec 2024-12-03T15:57:00,816 INFO [master/713339a81dd9:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733241420816, completionTime=-1 2024-12-03T15:57:00,820 INFO [master/713339a81dd9:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=3; waited=0ms, expected min=3 server(s), max=3 server(s), master is running 2024-12-03T15:57:00,821 DEBUG [master/713339a81dd9:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-03T15:57:00,925 INFO [master/713339a81dd9:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=3 2024-12-03T15:57:00,925 INFO [master/713339a81dd9:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733241480925 2024-12-03T15:57:00,944 INFO [master/713339a81dd9:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733241540925 2024-12-03T15:57:00,944 INFO [master/713339a81dd9:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 123 msec 2024-12-03T15:57:00,946 DEBUG [master/713339a81dd9:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-12-03T15:57:00,955 INFO [master/713339a81dd9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=713339a81dd9,34663,1733241417250-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-03T15:57:00,956 INFO [master/713339a81dd9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=713339a81dd9,34663,1733241417250-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-03T15:57:00,956 INFO [master/713339a81dd9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=713339a81dd9,34663,1733241417250-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-03T15:57:00,958 INFO [master/713339a81dd9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-713339a81dd9:34663, period=300000, unit=MILLISECONDS is enabled. 2024-12-03T15:57:00,958 INFO [master/713339a81dd9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-12-03T15:57:00,963 INFO [master/713339a81dd9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-12-03T15:57:00,974 DEBUG [master/713339a81dd9:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-12-03T15:57:00,996 INFO [master/713339a81dd9:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 2.475sec 2024-12-03T15:57:00,998 INFO [master/713339a81dd9:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-12-03T15:57:00,999 INFO [master/713339a81dd9:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-12-03T15:57:01,000 INFO [master/713339a81dd9:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-12-03T15:57:01,001 INFO [master/713339a81dd9:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-12-03T15:57:01,001 INFO [master/713339a81dd9:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-12-03T15:57:01,001 INFO [master/713339a81dd9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=713339a81dd9,34663,1733241417250-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-03T15:57:01,002 INFO [master/713339a81dd9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=713339a81dd9,34663,1733241417250-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-12-03T15:57:01,007 DEBUG [master/713339a81dd9:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-12-03T15:57:01,008 INFO [master/713339a81dd9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-12-03T15:57:01,009 INFO [master/713339a81dd9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=713339a81dd9,34663,1733241417250-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-03T15:57:01,058 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ce56598, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T15:57:01,059 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 713339a81dd9,34663,-1 for getting cluster id 2024-12-03T15:57:01,062 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T15:57:01,073 DEBUG [HMaster-EventLoopGroup-2-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a4fd42ae-14ff-4383-be19-b09963b80309' 2024-12-03T15:57:01,076 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T15:57:01,076 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a4fd42ae-14ff-4383-be19-b09963b80309" 2024-12-03T15:57:01,077 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6bd431fa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T15:57:01,077 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [713339a81dd9,34663,-1] 2024-12-03T15:57:01,081 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T15:57:01,083 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T15:57:01,084 INFO [HMaster-EventLoopGroup-2-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58568, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T15:57:01,087 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3f6303bc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T15:57:01,087 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T15:57:01,094 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=713339a81dd9,34819,1733241418314, seqNum=-1] 2024-12-03T15:57:01,095 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T15:57:01,097 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35184, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T15:57:01,127 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=713339a81dd9,34663,1733241417250 2024-12-03T15:57:01,127 INFO [Time-limited test {}] wal.AbstractTestWALReplay(147): hbase.rootdir=hdfs://localhost:45015/hbase 2024-12-03T15:57:01,148 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testNameConflictWhenSplit0 Thread=365, OpenFileDescriptor=607, MaxFileDescriptor=1048576, SystemLoadAverage=967, ProcessCount=11, AvailableMemoryMB=5371 2024-12-03T15:57:01,167 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-03T15:57:01,172 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-03T15:57:01,173 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-03T15:57:01,179 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-14996628, suffix=, logDir=hdfs://localhost:45015/hbase/WALs/hregion-14996628, archiveDir=hdfs://localhost:45015/hbase/oldWALs, maxLogs=32 2024-12-03T15:57:01,201 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-14996628/hregion-14996628.1733241421180, exclude list is [], retry=0 2024-12-03T15:57:01,207 DEBUG [AsyncFSWAL-8-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46585,DS-5c6771fd-75c7-4775-ad88-ae5bbe88de5c,DISK] 2024-12-03T15:57:01,208 DEBUG [AsyncFSWAL-8-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40575,DS-309fe9b7-1997-44cb-b0c9-1ce3c04a71de,DISK] 2024-12-03T15:57:01,208 DEBUG [AsyncFSWAL-8-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44827,DS-e007f1cc-fc1b-4368-9716-e11e4fe2f4eb,DISK] 2024-12-03T15:57:01,250 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-14996628/hregion-14996628.1733241421180 2024-12-03T15:57:01,250 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37355:37355),(127.0.0.1/127.0.0.1:44513:44513),(127.0.0.1/127.0.0.1:46731:46731)] 2024-12-03T15:57:01,250 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => b48243a131fb97f4ca00b08cb07d7682, NAME => 'testReplayEditsWrittenIntoWAL,,1733241421168.b48243a131fb97f4ca00b08cb07d7682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenIntoWAL', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45015/hbase 2024-12-03T15:57:01,268 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40575 is added to blk_1073741838_1014 (size=64) 2024-12-03T15:57:01,268 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44827 is added to blk_1073741838_1014 (size=64) 2024-12-03T15:57:01,269 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46585 is added to blk_1073741838_1014 (size=64) 2024-12-03T15:57:01,273 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1733241421168.b48243a131fb97f4ca00b08cb07d7682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T15:57:01,276 INFO [StoreOpener-b48243a131fb97f4ca00b08cb07d7682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region b48243a131fb97f4ca00b08cb07d7682 2024-12-03T15:57:01,279 INFO [StoreOpener-b48243a131fb97f4ca00b08cb07d7682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b48243a131fb97f4ca00b08cb07d7682 columnFamilyName a 2024-12-03T15:57:01,279 DEBUG [StoreOpener-b48243a131fb97f4ca00b08cb07d7682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T15:57:01,280 INFO [StoreOpener-b48243a131fb97f4ca00b08cb07d7682-1 {}] regionserver.HStore(327): Store=b48243a131fb97f4ca00b08cb07d7682/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T15:57:01,281 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for b48243a131fb97f4ca00b08cb07d7682 2024-12-03T15:57:01,283 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45015/hbase/data/default/testReplayEditsWrittenIntoWAL/b48243a131fb97f4ca00b08cb07d7682 2024-12-03T15:57:01,283 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45015/hbase/data/default/testReplayEditsWrittenIntoWAL/b48243a131fb97f4ca00b08cb07d7682 2024-12-03T15:57:01,284 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for b48243a131fb97f4ca00b08cb07d7682 2024-12-03T15:57:01,284 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for b48243a131fb97f4ca00b08cb07d7682 2024-12-03T15:57:01,288 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for b48243a131fb97f4ca00b08cb07d7682 2024-12-03T15:57:01,293 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45015/hbase/data/default/testReplayEditsWrittenIntoWAL/b48243a131fb97f4ca00b08cb07d7682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T15:57:01,294 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened b48243a131fb97f4ca00b08cb07d7682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74479165, jitterRate=0.10982604324817657}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-03T15:57:01,296 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for b48243a131fb97f4ca00b08cb07d7682: Writing region info on filesystem at 1733241421273Initializing all the Stores at 1733241421275 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733241421275Cleaning up temporary data from old regions at 1733241421284 (+9 ms)Region opened successfully at 1733241421296 (+12 ms) 2024-12-03T15:57:01,296 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing b48243a131fb97f4ca00b08cb07d7682, disabling compactions & flushes 2024-12-03T15:57:01,296 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1733241421168.b48243a131fb97f4ca00b08cb07d7682. 2024-12-03T15:57:01,296 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1733241421168.b48243a131fb97f4ca00b08cb07d7682. 2024-12-03T15:57:01,297 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1733241421168.b48243a131fb97f4ca00b08cb07d7682. after waiting 0 ms 2024-12-03T15:57:01,297 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1733241421168.b48243a131fb97f4ca00b08cb07d7682. 2024-12-03T15:57:01,297 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1733241421168.b48243a131fb97f4ca00b08cb07d7682. 2024-12-03T15:57:01,297 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for b48243a131fb97f4ca00b08cb07d7682: Waiting for close lock at 1733241421296Disabling compacts and flushes for region at 1733241421296Disabling writes for close at 1733241421297 (+1 ms)Writing region close event to WAL at 1733241421297Closed at 1733241421297 2024-12-03T15:57:01,310 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44827 is added to blk_1073741837_1013 (size=175) 2024-12-03T15:57:01,311 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /hbase/WALs/hregion-14996628/hregion-14996628.1733241421180 not finished, retry = 0 2024-12-03T15:57:01,311 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46585 is added to blk_1073741837_1013 (size=175) 2024-12-03T15:57:01,314 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40575 is added to blk_1073741837_1013 (size=175) 2024-12-03T15:57:01,419 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-03T15:57:01,419 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-14996628:(num 1733241421180) 2024-12-03T15:57:01,421 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-12-03T15:57:01,477 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40575 is added to blk_1073741839_1015 (size=469) 2024-12-03T15:57:01,477 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44827 is added to blk_1073741839_1015 (size=469) 2024-12-03T15:57:01,478 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46585 is added to blk_1073741839_1015 (size=469) 2024-12-03T15:57:01,481 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-12-03T15:57:01,526 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44827 is added to blk_1073741840_1016 (size=322) 2024-12-03T15:57:01,526 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40575 is added to blk_1073741840_1016 (size=322) 2024-12-03T15:57:01,529 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46585 is added to blk_1073741840_1016 (size=322) 2024-12-03T15:57:01,555 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:45015/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733241421166/wal-1, size=469 (469bytes) 2024-12-03T15:57:01,556 DEBUG [Time-limited test {}] util.RecoverLeaseFSUtils(47): Initialize RecoverLeaseFSUtils 2024-12-03T15:57:01,556 DEBUG [Time-limited test {}] util.RecoverLeaseFSUtils(59): set recoverLeaseMethod to org.apache.hadoop.fs.LeaseRecoverable.recoverLease() 2024-12-03T15:57:01,556 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:45015/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733241421166/wal-1 2024-12-03T15:57:01,564 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:45015/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733241421166/wal-1 after 5ms 2024-12-03T15:57:01,616 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-03T15:57:01,617 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:45015/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733241421166/wal-1 took 63ms 2024-12-03T15:57:01,624 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:45015/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733241421166/wal-1 so closing down 2024-12-03T15:57:01,624 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-03T15:57:01,628 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal-1.temp 2024-12-03T15:57:01,671 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:45015/hbase/data/default/testReplayEditsWrittenIntoWAL/b48243a131fb97f4ca00b08cb07d7682/recovered.edits/0000000000000000001-wal-1.temp 2024-12-03T15:57:01,672 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-03T15:57:01,684 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44827 is added to blk_1073741841_1017 (size=469) 2024-12-03T15:57:01,685 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46585 is added to blk_1073741841_1017 (size=469) 2024-12-03T15:57:01,685 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40575 is added to blk_1073741841_1017 (size=469) 2024-12-03T15:57:01,690 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:45015/hbase/data/default/testReplayEditsWrittenIntoWAL/b48243a131fb97f4ca00b08cb07d7682/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) 2024-12-03T15:57:01,694 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:45015/hbase/data/default/testReplayEditsWrittenIntoWAL/b48243a131fb97f4ca00b08cb07d7682/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:45015/hbase/data/default/testReplayEditsWrittenIntoWAL/b48243a131fb97f4ca00b08cb07d7682/recovered.edits/0000000000000000002 2024-12-03T15:57:01,699 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 2 edits across 1 Regions in 75 ms; skipped=0; WAL=hdfs://localhost:45015/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733241421166/wal-1, size=469, length=469, corrupted=false, cancelled=false 2024-12-03T15:57:01,700 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:45015/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733241421166/wal-1, journal: Splitting hdfs://localhost:45015/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733241421166/wal-1, size=469 (469bytes) at 1733241421555Finishing writing output for hdfs://localhost:45015/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733241421166/wal-1 so closing down at 1733241421624 (+69 ms)Creating recovered edits writer path=hdfs://localhost:45015/hbase/data/default/testReplayEditsWrittenIntoWAL/b48243a131fb97f4ca00b08cb07d7682/recovered.edits/0000000000000000001-wal-1.temp at 1733241421671 (+47 ms)3 split writer threads finished at 1733241421672 (+1 ms)Closed recovered edits writer path=hdfs://localhost:45015/hbase/data/default/testReplayEditsWrittenIntoWAL/b48243a131fb97f4ca00b08cb07d7682/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) at 1733241421690 (+18 ms)Rename recovered edits hdfs://localhost:45015/hbase/data/default/testReplayEditsWrittenIntoWAL/b48243a131fb97f4ca00b08cb07d7682/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:45015/hbase/data/default/testReplayEditsWrittenIntoWAL/b48243a131fb97f4ca00b08cb07d7682/recovered.edits/0000000000000000002 at 1733241421694 (+4 ms)Processed 2 edits across 1 Regions in 75 ms; skipped=0; WAL=hdfs://localhost:45015/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733241421166/wal-1, size=469, length=469, corrupted=false, cancelled=false at 1733241421700 (+6 ms) 2024-12-03T15:57:01,725 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:45015/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733241421166/wal-2, size=322 (322bytes) 2024-12-03T15:57:01,725 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:45015/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733241421166/wal-2 2024-12-03T15:57:01,726 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:45015/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733241421166/wal-2 after 1ms 2024-12-03T15:57:01,766 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-03T15:57:01,766 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:45015/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733241421166/wal-2 took 42ms 2024-12-03T15:57:01,769 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:45015/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733241421166/wal-2 so closing down 2024-12-03T15:57:01,769 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-03T15:57:01,773 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000002-wal-2.temp 2024-12-03T15:57:01,815 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:45015/hbase/data/default/testReplayEditsWrittenIntoWAL/b48243a131fb97f4ca00b08cb07d7682/recovered.edits/0000000000000000002-wal-2.temp 2024-12-03T15:57:01,816 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-03T15:57:01,829 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44827 is added to blk_1073741842_1018 (size=322) 2024-12-03T15:57:01,830 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40575 is added to blk_1073741842_1018 (size=322) 2024-12-03T15:57:01,831 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46585 is added to blk_1073741842_1018 (size=322) 2024-12-03T15:57:01,833 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:45015/hbase/data/default/testReplayEditsWrittenIntoWAL/b48243a131fb97f4ca00b08cb07d7682/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-12-03T15:57:01,876 DEBUG [split-log-closeStream-pool-0 {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-03T15:57:01,880 WARN [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(184): Found existing old edits file and we have less entries. Deleting hdfs://localhost:45015/hbase/data/default/testReplayEditsWrittenIntoWAL/b48243a131fb97f4ca00b08cb07d7682/recovered.edits/0000000000000000002-wal-2.temp, length=322 2024-12-03T15:57:01,883 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 1 edits across 1 Regions in 117 ms; skipped=0; WAL=hdfs://localhost:45015/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733241421166/wal-2, size=322, length=322, corrupted=false, cancelled=false 2024-12-03T15:57:01,883 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:45015/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733241421166/wal-2, journal: Splitting hdfs://localhost:45015/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733241421166/wal-2, size=322 (322bytes) at 1733241421725Finishing writing output for hdfs://localhost:45015/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733241421166/wal-2 so closing down at 1733241421769 (+44 ms)Creating recovered edits writer path=hdfs://localhost:45015/hbase/data/default/testReplayEditsWrittenIntoWAL/b48243a131fb97f4ca00b08cb07d7682/recovered.edits/0000000000000000002-wal-2.temp at 1733241421816 (+47 ms)3 split writer threads finished at 1733241421816Closed recovered edits writer path=hdfs://localhost:45015/hbase/data/default/testReplayEditsWrittenIntoWAL/b48243a131fb97f4ca00b08cb07d7682/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1733241421834 (+18 ms)Processed 1 edits across 1 Regions in 117 ms; skipped=0; WAL=hdfs://localhost:45015/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733241421166/wal-2, size=322, length=322, corrupted=false, cancelled=false at 1733241421883 (+49 ms) 2024-12-03T15:57:01,884 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-03T15:57:01,887 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:45015/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733241421166, archiveDir=hdfs://localhost:45015/hbase/oldWALs, maxLogs=32 2024-12-03T15:57:01,906 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testnameconflictwhensplit0-manual,16010,1733241421166/wal.1733241421888, exclude list is [], retry=0 2024-12-03T15:57:01,912 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44827,DS-e007f1cc-fc1b-4368-9716-e11e4fe2f4eb,DISK] 2024-12-03T15:57:01,913 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46585,DS-5c6771fd-75c7-4775-ad88-ae5bbe88de5c,DISK] 2024-12-03T15:57:01,913 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40575,DS-309fe9b7-1997-44cb-b0c9-1ce3c04a71de,DISK] 2024-12-03T15:57:01,955 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testnameconflictwhensplit0-manual,16010,1733241421166/wal.1733241421888 2024-12-03T15:57:01,961 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46731:46731),(127.0.0.1/127.0.0.1:37355:37355),(127.0.0.1/127.0.0.1:44513:44513)] 2024-12-03T15:57:01,961 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => b48243a131fb97f4ca00b08cb07d7682, NAME => 'testReplayEditsWrittenIntoWAL,,1733241421168.b48243a131fb97f4ca00b08cb07d7682.', STARTKEY => '', ENDKEY => ''} 2024-12-03T15:57:01,962 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1733241421168.b48243a131fb97f4ca00b08cb07d7682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T15:57:01,962 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for b48243a131fb97f4ca00b08cb07d7682 2024-12-03T15:57:01,962 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for b48243a131fb97f4ca00b08cb07d7682 2024-12-03T15:57:01,966 INFO [StoreOpener-b48243a131fb97f4ca00b08cb07d7682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region b48243a131fb97f4ca00b08cb07d7682 2024-12-03T15:57:01,968 INFO [StoreOpener-b48243a131fb97f4ca00b08cb07d7682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b48243a131fb97f4ca00b08cb07d7682 columnFamilyName a 2024-12-03T15:57:01,968 DEBUG [StoreOpener-b48243a131fb97f4ca00b08cb07d7682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T15:57:01,970 INFO [StoreOpener-b48243a131fb97f4ca00b08cb07d7682-1 {}] regionserver.HStore(327): Store=b48243a131fb97f4ca00b08cb07d7682/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T15:57:01,970 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for b48243a131fb97f4ca00b08cb07d7682 2024-12-03T15:57:01,974 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45015/hbase/data/default/testReplayEditsWrittenIntoWAL/b48243a131fb97f4ca00b08cb07d7682 2024-12-03T15:57:01,979 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:45015/hbase/data/default/testReplayEditsWrittenIntoWAL/b48243a131fb97f4ca00b08cb07d7682 2024-12-03T15:57:01,981 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:45015/hbase/data/default/testReplayEditsWrittenIntoWAL/b48243a131fb97f4ca00b08cb07d7682/recovered.edits/0000000000000000002 2024-12-03T15:57:02,017 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-03T15:57:02,024 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 2, skipped 0, firstSequenceIdInLog=1, maxSequenceIdInLog=2, path=hdfs://localhost:45015/hbase/data/default/testReplayEditsWrittenIntoWAL/b48243a131fb97f4ca00b08cb07d7682/recovered.edits/0000000000000000002 2024-12-03T15:57:02,028 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing b48243a131fb97f4ca00b08cb07d7682 1/1 column families, dataSize=108 B heapSize=512 B 2024-12-03T15:57:02,090 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46585 is added to blk_1073741829_1005 (size=34) 2024-12-03T15:57:02,104 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45015/hbase/data/default/testReplayEditsWrittenIntoWAL/b48243a131fb97f4ca00b08cb07d7682/.tmp/a/4d2e73f7bbef49e4aa04afb50e6b3b0b is 58, key is testReplayEditsWrittenIntoWAL/a:1/1733241421419/Put/seqid=0 2024-12-03T15:57:02,124 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40575 is added to blk_1073741844_1020 (size=5170) 2024-12-03T15:57:02,126 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44827 is added to blk_1073741844_1020 (size=5170) 2024-12-03T15:57:02,127 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46585 is added to blk_1073741844_1020 (size=5170) 2024-12-03T15:57:02,128 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=108 B at sequenceid=2 (bloomFilter=true), to=hdfs://localhost:45015/hbase/data/default/testReplayEditsWrittenIntoWAL/b48243a131fb97f4ca00b08cb07d7682/.tmp/a/4d2e73f7bbef49e4aa04afb50e6b3b0b 2024-12-03T15:57:02,181 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45015/hbase/data/default/testReplayEditsWrittenIntoWAL/b48243a131fb97f4ca00b08cb07d7682/.tmp/a/4d2e73f7bbef49e4aa04afb50e6b3b0b as hdfs://localhost:45015/hbase/data/default/testReplayEditsWrittenIntoWAL/b48243a131fb97f4ca00b08cb07d7682/a/4d2e73f7bbef49e4aa04afb50e6b3b0b 2024-12-03T15:57:02,191 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45015/hbase/data/default/testReplayEditsWrittenIntoWAL/b48243a131fb97f4ca00b08cb07d7682/a/4d2e73f7bbef49e4aa04afb50e6b3b0b, entries=2, sequenceid=2, filesize=5.0 K 2024-12-03T15:57:02,197 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for b48243a131fb97f4ca00b08cb07d7682 in 168ms, sequenceid=2, compaction requested=false; wal=null 2024-12-03T15:57:02,199 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:45015/hbase/data/default/testReplayEditsWrittenIntoWAL/b48243a131fb97f4ca00b08cb07d7682/recovered.edits/0000000000000000002 2024-12-03T15:57:02,200 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for b48243a131fb97f4ca00b08cb07d7682 2024-12-03T15:57:02,200 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for b48243a131fb97f4ca00b08cb07d7682 2024-12-03T15:57:02,203 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for b48243a131fb97f4ca00b08cb07d7682 2024-12-03T15:57:02,207 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45015/hbase/data/default/testReplayEditsWrittenIntoWAL/b48243a131fb97f4ca00b08cb07d7682/recovered.edits/2.seqid, newMaxSeqId=2, maxSeqId=1 2024-12-03T15:57:02,209 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened b48243a131fb97f4ca00b08cb07d7682; next sequenceid=3; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64390538, jitterRate=-0.040506213903427124}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-03T15:57:02,210 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for b48243a131fb97f4ca00b08cb07d7682: Writing region info on filesystem at 1733241421962Initializing all the Stores at 1733241421966 (+4 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733241421966Obtaining lock to block concurrent updates at 1733241422028 (+62 ms)Preparing flush snapshotting stores in b48243a131fb97f4ca00b08cb07d7682 at 1733241422028Finished memstore snapshotting testReplayEditsWrittenIntoWAL,,1733241421168.b48243a131fb97f4ca00b08cb07d7682., syncing WAL and waiting on mvcc, flushsize=dataSize=108, getHeapSize=496, getOffHeapSize=0, getCellsCount=2 at 1733241422031 (+3 ms)Flushing stores of testReplayEditsWrittenIntoWAL,,1733241421168.b48243a131fb97f4ca00b08cb07d7682. at 1733241422032 (+1 ms)Flushing b48243a131fb97f4ca00b08cb07d7682/a: creating writer at 1733241422033 (+1 ms)Flushing b48243a131fb97f4ca00b08cb07d7682/a: appending metadata at 1733241422089 (+56 ms)Flushing b48243a131fb97f4ca00b08cb07d7682/a: closing flushed file at 1733241422093 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5adc11ff: reopening flushed file at 1733241422179 (+86 ms)Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for b48243a131fb97f4ca00b08cb07d7682 in 168ms, sequenceid=2, compaction requested=false; wal=null at 1733241422197 (+18 ms)Cleaning up temporary data from old regions at 1733241422200 (+3 ms)Region opened successfully at 1733241422210 (+10 ms) 2024-12-03T15:57:02,246 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testNameConflictWhenSplit0 Thread=378 (was 365) Potentially hanging thread: AsyncFSWAL-0-hdfs://localhost:45015/hbase-prefix:default java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-8-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-8-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1122350750-172.17.0.3-1733241413817:blk_1073741843_1019, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-365845159_22 at /127.0.0.1:53606 [Waiting for operation #10] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: TestAsyncWALReplay-pool-0 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-8-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-365845159_22 at /127.0.0.1:50872 [Waiting for operation #6] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-365845159_22 at /127.0.0.1:40364 [Receiving block BP-1122350750-172.17.0.3-1733241413817:blk_1073741843_1019] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1122350750-172.17.0.3-1733241413817:blk_1073741843_1019, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-365845159_22 at /127.0.0.1:43700 [Waiting for operation #6] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1122350750-172.17.0.3-1733241413817:blk_1073741843_1019, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-365845159_22 at /127.0.0.1:50640 [Receiving block BP-1122350750-172.17.0.3-1733241413817:blk_1073741843_1019] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-365845159_22 at /127.0.0.1:53594 [Receiving block BP-1122350750-172.17.0.3-1733241413817:blk_1073741843_1019] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: java.util.concurrent.ThreadPoolExecutor$Worker@511f56a9[State = -1, empty queue] java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:401) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=703 (was 607) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=921 (was 967), ProcessCount=11 (was 11), AvailableMemoryMB=5279 (was 5371) 2024-12-03T15:57:02,264 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testNameConflictWhenSplit1 Thread=378, OpenFileDescriptor=703, MaxFileDescriptor=1048576, SystemLoadAverage=921, ProcessCount=11, AvailableMemoryMB=5277 2024-12-03T15:57:02,284 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-03T15:57:02,287 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-03T15:57:02,288 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-03T15:57:02,293 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-71356178, suffix=, logDir=hdfs://localhost:45015/hbase/WALs/hregion-71356178, archiveDir=hdfs://localhost:45015/hbase/oldWALs, maxLogs=32 2024-12-03T15:57:02,308 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-71356178/hregion-71356178.1733241422294, exclude list is [], retry=0 2024-12-03T15:57:02,313 DEBUG [AsyncFSWAL-10-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40575,DS-309fe9b7-1997-44cb-b0c9-1ce3c04a71de,DISK] 2024-12-03T15:57:02,314 DEBUG [AsyncFSWAL-10-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44827,DS-e007f1cc-fc1b-4368-9716-e11e4fe2f4eb,DISK] 2024-12-03T15:57:02,317 DEBUG [AsyncFSWAL-10-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46585,DS-5c6771fd-75c7-4775-ad88-ae5bbe88de5c,DISK] 2024-12-03T15:57:02,354 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-71356178/hregion-71356178.1733241422294 2024-12-03T15:57:02,354 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44513:44513),(127.0.0.1/127.0.0.1:46731:46731),(127.0.0.1/127.0.0.1:37355:37355)] 2024-12-03T15:57:02,355 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => dad5782dbb4aca58c4273cb0999899d2, NAME => 'testReplayEditsWrittenIntoWAL,,1733241422285.dad5782dbb4aca58c4273cb0999899d2.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenIntoWAL', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45015/hbase 2024-12-03T15:57:02,373 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46585 is added to blk_1073741846_1022 (size=64) 2024-12-03T15:57:02,375 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40575 is added to blk_1073741846_1022 (size=64) 2024-12-03T15:57:02,375 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44827 is added to blk_1073741846_1022 (size=64) 2024-12-03T15:57:02,375 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1733241422285.dad5782dbb4aca58c4273cb0999899d2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T15:57:02,378 INFO [StoreOpener-dad5782dbb4aca58c4273cb0999899d2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region dad5782dbb4aca58c4273cb0999899d2 2024-12-03T15:57:02,386 INFO [StoreOpener-dad5782dbb4aca58c4273cb0999899d2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region dad5782dbb4aca58c4273cb0999899d2 columnFamilyName a 2024-12-03T15:57:02,386 DEBUG [StoreOpener-dad5782dbb4aca58c4273cb0999899d2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T15:57:02,388 INFO [StoreOpener-dad5782dbb4aca58c4273cb0999899d2-1 {}] regionserver.HStore(327): Store=dad5782dbb4aca58c4273cb0999899d2/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T15:57:02,388 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for dad5782dbb4aca58c4273cb0999899d2 2024-12-03T15:57:02,390 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45015/hbase/data/default/testReplayEditsWrittenIntoWAL/dad5782dbb4aca58c4273cb0999899d2 2024-12-03T15:57:02,392 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45015/hbase/data/default/testReplayEditsWrittenIntoWAL/dad5782dbb4aca58c4273cb0999899d2 2024-12-03T15:57:02,392 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for dad5782dbb4aca58c4273cb0999899d2 2024-12-03T15:57:02,392 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for dad5782dbb4aca58c4273cb0999899d2 2024-12-03T15:57:02,395 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for dad5782dbb4aca58c4273cb0999899d2 2024-12-03T15:57:02,400 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45015/hbase/data/default/testReplayEditsWrittenIntoWAL/dad5782dbb4aca58c4273cb0999899d2/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T15:57:02,401 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened dad5782dbb4aca58c4273cb0999899d2; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68965368, jitterRate=0.02766406536102295}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-03T15:57:02,401 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for dad5782dbb4aca58c4273cb0999899d2: Writing region info on filesystem at 1733241422376Initializing all the Stores at 1733241422377 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733241422377Cleaning up temporary data from old regions at 1733241422392 (+15 ms)Region opened successfully at 1733241422401 (+9 ms) 2024-12-03T15:57:02,401 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing dad5782dbb4aca58c4273cb0999899d2, disabling compactions & flushes 2024-12-03T15:57:02,401 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1733241422285.dad5782dbb4aca58c4273cb0999899d2. 2024-12-03T15:57:02,401 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1733241422285.dad5782dbb4aca58c4273cb0999899d2. 2024-12-03T15:57:02,402 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1733241422285.dad5782dbb4aca58c4273cb0999899d2. after waiting 0 ms 2024-12-03T15:57:02,402 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1733241422285.dad5782dbb4aca58c4273cb0999899d2. 2024-12-03T15:57:02,402 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1733241422285.dad5782dbb4aca58c4273cb0999899d2. 2024-12-03T15:57:02,402 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for dad5782dbb4aca58c4273cb0999899d2: Waiting for close lock at 1733241422401Disabling compacts and flushes for region at 1733241422401Disabling writes for close at 1733241422402 (+1 ms)Writing region close event to WAL at 1733241422402Closed at 1733241422402 2024-12-03T15:57:02,410 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44827 is added to blk_1073741845_1021 (size=175) 2024-12-03T15:57:02,410 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46585 is added to blk_1073741845_1021 (size=175) 2024-12-03T15:57:02,411 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40575 is added to blk_1073741845_1021 (size=175) 2024-12-03T15:57:02,423 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-03T15:57:02,424 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-71356178:(num 1733241422294) 2024-12-03T15:57:02,426 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-12-03T15:57:02,475 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46585 is added to blk_1073741847_1023 (size=469) 2024-12-03T15:57:02,479 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40575 is added to blk_1073741847_1023 (size=469) 2024-12-03T15:57:02,480 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44827 is added to blk_1073741847_1023 (size=469) 2024-12-03T15:57:02,482 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-12-03T15:57:02,543 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40575 is added to blk_1073741848_1024 (size=322) 2024-12-03T15:57:02,543 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46585 is added to blk_1073741848_1024 (size=322) 2024-12-03T15:57:02,544 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44827 is added to blk_1073741848_1024 (size=322) 2024-12-03T15:57:02,564 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:45015/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733241422283/wal-2, size=322 (322bytes) 2024-12-03T15:57:02,564 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:45015/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733241422283/wal-2 2024-12-03T15:57:02,565 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:45015/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733241422283/wal-2 after 1ms 2024-12-03T15:57:02,612 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-03T15:57:02,612 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:45015/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733241422283/wal-2 took 49ms 2024-12-03T15:57:02,618 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:45015/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733241422283/wal-2 so closing down 2024-12-03T15:57:02,618 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-03T15:57:02,622 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000002-wal-2.temp 2024-12-03T15:57:02,667 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:45015/hbase/data/default/testReplayEditsWrittenIntoWAL/dad5782dbb4aca58c4273cb0999899d2/recovered.edits/0000000000000000002-wal-2.temp 2024-12-03T15:57:02,668 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-03T15:57:02,680 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46585 is added to blk_1073741849_1025 (size=322) 2024-12-03T15:57:02,681 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40575 is added to blk_1073741849_1025 (size=322) 2024-12-03T15:57:02,681 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44827 is added to blk_1073741849_1025 (size=322) 2024-12-03T15:57:02,682 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:45015/hbase/data/default/testReplayEditsWrittenIntoWAL/dad5782dbb4aca58c4273cb0999899d2/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-12-03T15:57:02,684 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:45015/hbase/data/default/testReplayEditsWrittenIntoWAL/dad5782dbb4aca58c4273cb0999899d2/recovered.edits/0000000000000000002-wal-2.temp to hdfs://localhost:45015/hbase/data/default/testReplayEditsWrittenIntoWAL/dad5782dbb4aca58c4273cb0999899d2/recovered.edits/0000000000000000002 2024-12-03T15:57:02,685 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 1 edits across 1 Regions in 73 ms; skipped=0; WAL=hdfs://localhost:45015/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733241422283/wal-2, size=322, length=322, corrupted=false, cancelled=false 2024-12-03T15:57:02,685 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:45015/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733241422283/wal-2, journal: Splitting hdfs://localhost:45015/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733241422283/wal-2, size=322 (322bytes) at 1733241422564Finishing writing output for hdfs://localhost:45015/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733241422283/wal-2 so closing down at 1733241422618 (+54 ms)Creating recovered edits writer path=hdfs://localhost:45015/hbase/data/default/testReplayEditsWrittenIntoWAL/dad5782dbb4aca58c4273cb0999899d2/recovered.edits/0000000000000000002-wal-2.temp at 1733241422667 (+49 ms)3 split writer threads finished at 1733241422668 (+1 ms)Closed recovered edits writer path=hdfs://localhost:45015/hbase/data/default/testReplayEditsWrittenIntoWAL/dad5782dbb4aca58c4273cb0999899d2/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1733241422682 (+14 ms)Rename recovered edits hdfs://localhost:45015/hbase/data/default/testReplayEditsWrittenIntoWAL/dad5782dbb4aca58c4273cb0999899d2/recovered.edits/0000000000000000002-wal-2.temp to hdfs://localhost:45015/hbase/data/default/testReplayEditsWrittenIntoWAL/dad5782dbb4aca58c4273cb0999899d2/recovered.edits/0000000000000000002 at 1733241422684 (+2 ms)Processed 1 edits across 1 Regions in 73 ms; skipped=0; WAL=hdfs://localhost:45015/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733241422283/wal-2, size=322, length=322, corrupted=false, cancelled=false at 1733241422685 (+1 ms) 2024-12-03T15:57:02,708 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:45015/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733241422283/wal-1, size=469 (469bytes) 2024-12-03T15:57:02,708 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:45015/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733241422283/wal-1 2024-12-03T15:57:02,709 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:45015/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733241422283/wal-1 after 0ms 2024-12-03T15:57:02,740 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-03T15:57:02,740 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:45015/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733241422283/wal-1 took 33ms 2024-12-03T15:57:02,745 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:45015/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733241422283/wal-1 so closing down 2024-12-03T15:57:02,745 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-03T15:57:02,748 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal-1.temp 2024-12-03T15:57:02,787 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:45015/hbase/data/default/testReplayEditsWrittenIntoWAL/dad5782dbb4aca58c4273cb0999899d2/recovered.edits/0000000000000000001-wal-1.temp 2024-12-03T15:57:02,787 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-03T15:57:02,805 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44827 is added to blk_1073741850_1026 (size=469) 2024-12-03T15:57:02,805 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46585 is added to blk_1073741850_1026 (size=469) 2024-12-03T15:57:02,805 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40575 is added to blk_1073741850_1026 (size=469) 2024-12-03T15:57:03,202 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:45015/hbase/data/default/testReplayEditsWrittenIntoWAL/dad5782dbb4aca58c4273cb0999899d2/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) 2024-12-03T15:57:03,246 DEBUG [split-log-closeStream-pool-0 {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-03T15:57:03,250 WARN [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(175): Found existing old edits file. It could be the result of a previous failed split attempt or we have duplicated wal entries. Deleting hdfs://localhost:45015/hbase/data/default/testReplayEditsWrittenIntoWAL/dad5782dbb4aca58c4273cb0999899d2/recovered.edits/0000000000000000002, length=322 2024-12-03T15:57:03,253 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:45015/hbase/data/default/testReplayEditsWrittenIntoWAL/dad5782dbb4aca58c4273cb0999899d2/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:45015/hbase/data/default/testReplayEditsWrittenIntoWAL/dad5782dbb4aca58c4273cb0999899d2/recovered.edits/0000000000000000002 2024-12-03T15:57:03,253 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 2 edits across 1 Regions in 511 ms; skipped=0; WAL=hdfs://localhost:45015/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733241422283/wal-1, size=469, length=469, corrupted=false, cancelled=false 2024-12-03T15:57:03,253 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:45015/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733241422283/wal-1, journal: Splitting hdfs://localhost:45015/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733241422283/wal-1, size=469 (469bytes) at 1733241422708Finishing writing output for hdfs://localhost:45015/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733241422283/wal-1 so closing down at 1733241422745 (+37 ms)Creating recovered edits writer path=hdfs://localhost:45015/hbase/data/default/testReplayEditsWrittenIntoWAL/dad5782dbb4aca58c4273cb0999899d2/recovered.edits/0000000000000000001-wal-1.temp at 1733241422787 (+42 ms)3 split writer threads finished at 1733241422787Closed recovered edits writer path=hdfs://localhost:45015/hbase/data/default/testReplayEditsWrittenIntoWAL/dad5782dbb4aca58c4273cb0999899d2/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) at 1733241423202 (+415 ms)Rename recovered edits hdfs://localhost:45015/hbase/data/default/testReplayEditsWrittenIntoWAL/dad5782dbb4aca58c4273cb0999899d2/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:45015/hbase/data/default/testReplayEditsWrittenIntoWAL/dad5782dbb4aca58c4273cb0999899d2/recovered.edits/0000000000000000002 at 1733241423253 (+51 ms)Processed 2 edits across 1 Regions in 511 ms; skipped=0; WAL=hdfs://localhost:45015/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733241422283/wal-1, size=469, length=469, corrupted=false, cancelled=false at 1733241423253 2024-12-03T15:57:03,253 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-03T15:57:03,256 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:45015/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733241422283, archiveDir=hdfs://localhost:45015/hbase/oldWALs, maxLogs=32 2024-12-03T15:57:03,270 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testnameconflictwhensplit1-manual,16010,1733241422283/wal.1733241423256, exclude list is [], retry=0 2024-12-03T15:57:03,274 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40575,DS-309fe9b7-1997-44cb-b0c9-1ce3c04a71de,DISK] 2024-12-03T15:57:03,274 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44827,DS-e007f1cc-fc1b-4368-9716-e11e4fe2f4eb,DISK] 2024-12-03T15:57:03,275 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46585,DS-5c6771fd-75c7-4775-ad88-ae5bbe88de5c,DISK] 2024-12-03T15:57:03,305 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testnameconflictwhensplit1-manual,16010,1733241422283/wal.1733241423256 2024-12-03T15:57:03,305 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44513:44513),(127.0.0.1/127.0.0.1:46731:46731),(127.0.0.1/127.0.0.1:37355:37355)] 2024-12-03T15:57:03,305 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => dad5782dbb4aca58c4273cb0999899d2, NAME => 'testReplayEditsWrittenIntoWAL,,1733241422285.dad5782dbb4aca58c4273cb0999899d2.', STARTKEY => '', ENDKEY => ''} 2024-12-03T15:57:03,306 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1733241422285.dad5782dbb4aca58c4273cb0999899d2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T15:57:03,306 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for dad5782dbb4aca58c4273cb0999899d2 2024-12-03T15:57:03,306 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for dad5782dbb4aca58c4273cb0999899d2 2024-12-03T15:57:03,308 INFO [StoreOpener-dad5782dbb4aca58c4273cb0999899d2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region dad5782dbb4aca58c4273cb0999899d2 2024-12-03T15:57:03,310 INFO [StoreOpener-dad5782dbb4aca58c4273cb0999899d2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region dad5782dbb4aca58c4273cb0999899d2 columnFamilyName a 2024-12-03T15:57:03,310 DEBUG [StoreOpener-dad5782dbb4aca58c4273cb0999899d2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T15:57:03,311 INFO [StoreOpener-dad5782dbb4aca58c4273cb0999899d2-1 {}] regionserver.HStore(327): Store=dad5782dbb4aca58c4273cb0999899d2/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T15:57:03,311 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for dad5782dbb4aca58c4273cb0999899d2 2024-12-03T15:57:03,312 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45015/hbase/data/default/testReplayEditsWrittenIntoWAL/dad5782dbb4aca58c4273cb0999899d2 2024-12-03T15:57:03,314 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:45015/hbase/data/default/testReplayEditsWrittenIntoWAL/dad5782dbb4aca58c4273cb0999899d2 2024-12-03T15:57:03,315 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:45015/hbase/data/default/testReplayEditsWrittenIntoWAL/dad5782dbb4aca58c4273cb0999899d2/recovered.edits/0000000000000000002 2024-12-03T15:57:03,345 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-03T15:57:03,347 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 2, skipped 0, firstSequenceIdInLog=1, maxSequenceIdInLog=2, path=hdfs://localhost:45015/hbase/data/default/testReplayEditsWrittenIntoWAL/dad5782dbb4aca58c4273cb0999899d2/recovered.edits/0000000000000000002 2024-12-03T15:57:03,347 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing dad5782dbb4aca58c4273cb0999899d2 1/1 column families, dataSize=108 B heapSize=512 B 2024-12-03T15:57:03,365 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45015/hbase/data/default/testReplayEditsWrittenIntoWAL/dad5782dbb4aca58c4273cb0999899d2/.tmp/a/b634d128f76b46869fcebd8401509864 is 58, key is testReplayEditsWrittenIntoWAL/a:1/1733241422424/Put/seqid=0 2024-12-03T15:57:03,374 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40575 is added to blk_1073741852_1028 (size=5170) 2024-12-03T15:57:03,374 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44827 is added to blk_1073741852_1028 (size=5170) 2024-12-03T15:57:03,375 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46585 is added to blk_1073741852_1028 (size=5170) 2024-12-03T15:57:03,375 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=108 B at sequenceid=2 (bloomFilter=true), to=hdfs://localhost:45015/hbase/data/default/testReplayEditsWrittenIntoWAL/dad5782dbb4aca58c4273cb0999899d2/.tmp/a/b634d128f76b46869fcebd8401509864 2024-12-03T15:57:03,384 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45015/hbase/data/default/testReplayEditsWrittenIntoWAL/dad5782dbb4aca58c4273cb0999899d2/.tmp/a/b634d128f76b46869fcebd8401509864 as hdfs://localhost:45015/hbase/data/default/testReplayEditsWrittenIntoWAL/dad5782dbb4aca58c4273cb0999899d2/a/b634d128f76b46869fcebd8401509864 2024-12-03T15:57:03,392 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45015/hbase/data/default/testReplayEditsWrittenIntoWAL/dad5782dbb4aca58c4273cb0999899d2/a/b634d128f76b46869fcebd8401509864, entries=2, sequenceid=2, filesize=5.0 K 2024-12-03T15:57:03,392 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for dad5782dbb4aca58c4273cb0999899d2 in 45ms, sequenceid=2, compaction requested=false; wal=null 2024-12-03T15:57:03,393 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:45015/hbase/data/default/testReplayEditsWrittenIntoWAL/dad5782dbb4aca58c4273cb0999899d2/recovered.edits/0000000000000000002 2024-12-03T15:57:03,394 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for dad5782dbb4aca58c4273cb0999899d2 2024-12-03T15:57:03,394 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for dad5782dbb4aca58c4273cb0999899d2 2024-12-03T15:57:03,396 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for dad5782dbb4aca58c4273cb0999899d2 2024-12-03T15:57:03,399 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45015/hbase/data/default/testReplayEditsWrittenIntoWAL/dad5782dbb4aca58c4273cb0999899d2/recovered.edits/2.seqid, newMaxSeqId=2, maxSeqId=1 2024-12-03T15:57:03,400 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened dad5782dbb4aca58c4273cb0999899d2; next sequenceid=3; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68430073, jitterRate=0.019687548279762268}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-03T15:57:03,400 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for dad5782dbb4aca58c4273cb0999899d2: Writing region info on filesystem at 1733241423306Initializing all the Stores at 1733241423308 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733241423308Obtaining lock to block concurrent updates at 1733241423347 (+39 ms)Preparing flush snapshotting stores in dad5782dbb4aca58c4273cb0999899d2 at 1733241423347Finished memstore snapshotting testReplayEditsWrittenIntoWAL,,1733241422285.dad5782dbb4aca58c4273cb0999899d2., syncing WAL and waiting on mvcc, flushsize=dataSize=108, getHeapSize=496, getOffHeapSize=0, getCellsCount=2 at 1733241423348 (+1 ms)Flushing stores of testReplayEditsWrittenIntoWAL,,1733241422285.dad5782dbb4aca58c4273cb0999899d2. at 1733241423348Flushing dad5782dbb4aca58c4273cb0999899d2/a: creating writer at 1733241423348Flushing dad5782dbb4aca58c4273cb0999899d2/a: appending metadata at 1733241423364 (+16 ms)Flushing dad5782dbb4aca58c4273cb0999899d2/a: closing flushed file at 1733241423364Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@21491daf: reopening flushed file at 1733241423383 (+19 ms)Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for dad5782dbb4aca58c4273cb0999899d2 in 45ms, sequenceid=2, compaction requested=false; wal=null at 1733241423392 (+9 ms)Cleaning up temporary data from old regions at 1733241423394 (+2 ms)Region opened successfully at 1733241423400 (+6 ms) 2024-12-03T15:57:03,417 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testNameConflictWhenSplit1 Thread=388 (was 378) Potentially hanging thread: PacketResponder: BP-1122350750-172.17.0.3-1733241413817:blk_1073741851_1027, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-10-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1122350750-172.17.0.3-1733241413817:blk_1073741851_1027, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-365845159_22 at /127.0.0.1:50872 [Waiting for operation #12] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-10-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-10-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-365845159_22 at /127.0.0.1:53606 [Waiting for operation #19] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1122350750-172.17.0.3-1733241413817:blk_1073741851_1027, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-365845159_22 at /127.0.0.1:40440 [Receiving block BP-1122350750-172.17.0.3-1733241413817:blk_1073741851_1027] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-365845159_22 at /127.0.0.1:50704 [Receiving block BP-1122350750-172.17.0.3-1733241413817:blk_1073741851_1027] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-365845159_22 at /127.0.0.1:53694 [Receiving block BP-1122350750-172.17.0.3-1733241413817:blk_1073741851_1027] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=785 (was 703) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=921 (was 921), ProcessCount=11 (was 11), AvailableMemoryMB=5362 (was 5277) - AvailableMemoryMB LEAK? - 2024-12-03T15:57:03,430 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsWrittenIntoWAL Thread=388, OpenFileDescriptor=785, MaxFileDescriptor=1048576, SystemLoadAverage=921, ProcessCount=11, AvailableMemoryMB=5361 2024-12-03T15:57:03,447 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-03T15:57:03,449 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-03T15:57:03,450 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-03T15:57:03,453 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-11139740, suffix=, logDir=hdfs://localhost:45015/hbase/WALs/hregion-11139740, archiveDir=hdfs://localhost:45015/hbase/oldWALs, maxLogs=32 2024-12-03T15:57:03,467 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-11139740/hregion-11139740.1733241423454, exclude list is [], retry=0 2024-12-03T15:57:03,471 DEBUG [AsyncFSWAL-12-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44827,DS-e007f1cc-fc1b-4368-9716-e11e4fe2f4eb,DISK] 2024-12-03T15:57:03,471 DEBUG [AsyncFSWAL-12-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46585,DS-5c6771fd-75c7-4775-ad88-ae5bbe88de5c,DISK] 2024-12-03T15:57:03,471 DEBUG [AsyncFSWAL-12-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40575,DS-309fe9b7-1997-44cb-b0c9-1ce3c04a71de,DISK] 2024-12-03T15:57:03,501 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-11139740/hregion-11139740.1733241423454 2024-12-03T15:57:03,502 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46731:46731),(127.0.0.1/127.0.0.1:37355:37355),(127.0.0.1/127.0.0.1:44513:44513)] 2024-12-03T15:57:03,502 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 0af130a07261dbe3a5b02b93d8b2bb99, NAME => 'testReplayEditsWrittenIntoWAL,,1733241423448.0af130a07261dbe3a5b02b93d8b2bb99.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenIntoWAL', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45015/hbase 2024-12-03T15:57:03,513 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44827 is added to blk_1073741854_1030 (size=64) 2024-12-03T15:57:03,513 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46585 is added to blk_1073741854_1030 (size=64) 2024-12-03T15:57:03,514 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40575 is added to blk_1073741854_1030 (size=64) 2024-12-03T15:57:03,515 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1733241423448.0af130a07261dbe3a5b02b93d8b2bb99.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T15:57:03,517 INFO [StoreOpener-0af130a07261dbe3a5b02b93d8b2bb99-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 0af130a07261dbe3a5b02b93d8b2bb99 2024-12-03T15:57:03,519 INFO [StoreOpener-0af130a07261dbe3a5b02b93d8b2bb99-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0af130a07261dbe3a5b02b93d8b2bb99 columnFamilyName a 2024-12-03T15:57:03,519 DEBUG [StoreOpener-0af130a07261dbe3a5b02b93d8b2bb99-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T15:57:03,520 INFO [StoreOpener-0af130a07261dbe3a5b02b93d8b2bb99-1 {}] regionserver.HStore(327): Store=0af130a07261dbe3a5b02b93d8b2bb99/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T15:57:03,520 INFO [StoreOpener-0af130a07261dbe3a5b02b93d8b2bb99-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 0af130a07261dbe3a5b02b93d8b2bb99 2024-12-03T15:57:03,522 INFO [StoreOpener-0af130a07261dbe3a5b02b93d8b2bb99-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0af130a07261dbe3a5b02b93d8b2bb99 columnFamilyName b 2024-12-03T15:57:03,522 DEBUG [StoreOpener-0af130a07261dbe3a5b02b93d8b2bb99-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T15:57:03,522 INFO [StoreOpener-0af130a07261dbe3a5b02b93d8b2bb99-1 {}] regionserver.HStore(327): Store=0af130a07261dbe3a5b02b93d8b2bb99/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T15:57:03,523 INFO [StoreOpener-0af130a07261dbe3a5b02b93d8b2bb99-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 0af130a07261dbe3a5b02b93d8b2bb99 2024-12-03T15:57:03,525 INFO [StoreOpener-0af130a07261dbe3a5b02b93d8b2bb99-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0af130a07261dbe3a5b02b93d8b2bb99 columnFamilyName c 2024-12-03T15:57:03,525 DEBUG [StoreOpener-0af130a07261dbe3a5b02b93d8b2bb99-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T15:57:03,526 INFO [StoreOpener-0af130a07261dbe3a5b02b93d8b2bb99-1 {}] regionserver.HStore(327): Store=0af130a07261dbe3a5b02b93d8b2bb99/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T15:57:03,526 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 0af130a07261dbe3a5b02b93d8b2bb99 2024-12-03T15:57:03,527 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45015/hbase/data/default/testReplayEditsWrittenIntoWAL/0af130a07261dbe3a5b02b93d8b2bb99 2024-12-03T15:57:03,527 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45015/hbase/data/default/testReplayEditsWrittenIntoWAL/0af130a07261dbe3a5b02b93d8b2bb99 2024-12-03T15:57:03,529 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 0af130a07261dbe3a5b02b93d8b2bb99 2024-12-03T15:57:03,529 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 0af130a07261dbe3a5b02b93d8b2bb99 2024-12-03T15:57:03,529 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenIntoWAL descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-03T15:57:03,531 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 0af130a07261dbe3a5b02b93d8b2bb99 2024-12-03T15:57:03,534 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45015/hbase/data/default/testReplayEditsWrittenIntoWAL/0af130a07261dbe3a5b02b93d8b2bb99/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T15:57:03,535 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 0af130a07261dbe3a5b02b93d8b2bb99; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64416485, jitterRate=-0.04011957347393036}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-03T15:57:03,535 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 0af130a07261dbe3a5b02b93d8b2bb99: Writing region info on filesystem at 1733241423515Initializing all the Stores at 1733241423516 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733241423517 (+1 ms)Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733241423517Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733241423517Cleaning up temporary data from old regions at 1733241423529 (+12 ms)Region opened successfully at 1733241423535 (+6 ms) 2024-12-03T15:57:03,535 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 0af130a07261dbe3a5b02b93d8b2bb99, disabling compactions & flushes 2024-12-03T15:57:03,535 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1733241423448.0af130a07261dbe3a5b02b93d8b2bb99. 2024-12-03T15:57:03,535 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1733241423448.0af130a07261dbe3a5b02b93d8b2bb99. 2024-12-03T15:57:03,535 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1733241423448.0af130a07261dbe3a5b02b93d8b2bb99. after waiting 0 ms 2024-12-03T15:57:03,535 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1733241423448.0af130a07261dbe3a5b02b93d8b2bb99. 2024-12-03T15:57:03,536 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1733241423448.0af130a07261dbe3a5b02b93d8b2bb99. 2024-12-03T15:57:03,536 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 0af130a07261dbe3a5b02b93d8b2bb99: Waiting for close lock at 1733241423535Disabling compacts and flushes for region at 1733241423535Disabling writes for close at 1733241423535Writing region close event to WAL at 1733241423536 (+1 ms)Closed at 1733241423536 2024-12-03T15:57:03,540 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44827 is added to blk_1073741853_1029 (size=175) 2024-12-03T15:57:03,540 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46585 is added to blk_1073741853_1029 (size=175) 2024-12-03T15:57:03,540 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40575 is added to blk_1073741853_1029 (size=175) 2024-12-03T15:57:03,543 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-03T15:57:03,543 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-11139740:(num 1733241423454) 2024-12-03T15:57:03,543 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-03T15:57:03,546 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:45015/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733241423446, archiveDir=hdfs://localhost:45015/hbase/oldWALs, maxLogs=32 2024-12-03T15:57:03,560 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733241423446/wal.1733241423546, exclude list is [], retry=0 2024-12-03T15:57:03,563 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46585,DS-5c6771fd-75c7-4775-ad88-ae5bbe88de5c,DISK] 2024-12-03T15:57:03,564 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44827,DS-e007f1cc-fc1b-4368-9716-e11e4fe2f4eb,DISK] 2024-12-03T15:57:03,564 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40575,DS-309fe9b7-1997-44cb-b0c9-1ce3c04a71de,DISK] 2024-12-03T15:57:03,594 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733241423446/wal.1733241423546 2024-12-03T15:57:03,594 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37355:37355),(127.0.0.1/127.0.0.1:46731:46731),(127.0.0.1/127.0.0.1:44513:44513)] 2024-12-03T15:57:03,789 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:45015/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733241423446/wal.1733241423546, size=0 (0bytes) 2024-12-03T15:57:03,789 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:45015/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733241423446/wal.1733241423546 might be still open, length is 0 2024-12-03T15:57:03,789 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:45015/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733241423446/wal.1733241423546 2024-12-03T15:57:03,790 WARN [IPC Server handler 1 on default port 45015 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733241423446/wal.1733241423546 has not been closed. Lease recovery is in progress. RecoveryId = 1032 for block blk_1073741855_1031 2024-12-03T15:57:03,791 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:45015/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733241423446/wal.1733241423546 after 2ms 2024-12-03T15:57:05,082 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-365845159_22 at /127.0.0.1:50746 [Receiving block BP-1122350750-172.17.0.3-1733241413817:blk_1073741855_1031] {}] datanode.DataXceiver(331): 127.0.0.1:46585:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:50746 dst: /127.0.0.1:46585 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:46585 remote=/127.0.0.1:50746]. Total timeout mills is 60000, 58673 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T15:57:05,083 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-365845159_22 at /127.0.0.1:53740 [Receiving block BP-1122350750-172.17.0.3-1733241413817:blk_1073741855_1031] {}] datanode.DataXceiver(331): 127.0.0.1:40575:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:53740 dst: /127.0.0.1:40575 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T15:57:05,083 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-365845159_22 at /127.0.0.1:40476 [Receiving block BP-1122350750-172.17.0.3-1733241413817:blk_1073741855_1031] {}] datanode.DataXceiver(331): 127.0.0.1:44827:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:40476 dst: /127.0.0.1:44827 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T15:57:05,092 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46585 is added to blk_1073741855_1032 (size=497775) 2024-12-03T15:57:05,094 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44827 is added to blk_1073741855_1032 (size=497775) 2024-12-03T15:57:05,095 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40575 is added to blk_1073741855_1032 (size=497775) 2024-12-03T15:57:06,135 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-03T15:57:06,204 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-12-03T15:57:07,779 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-12-03T15:57:07,780 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-12-03T15:57:07,781 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-03T15:57:07,781 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-12-03T15:57:07,781 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-12-03T15:57:07,781 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-12-03T15:57:07,782 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenIntoWAL 2024-12-03T15:57:07,782 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenIntoWAL Metrics about Tables on a single HBase RegionServer 2024-12-03T15:57:07,792 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:45015/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733241423446/wal.1733241423546 after 4003ms 2024-12-03T15:57:07,833 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-03T15:57:07,836 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:45015/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733241423446/wal.1733241423546 took 4047ms 2024-12-03T15:57:07,850 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal.1733241423546.temp 2024-12-03T15:57:07,904 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:45015/hbase/data/default/testReplayEditsWrittenIntoWAL/0af130a07261dbe3a5b02b93d8b2bb99/recovered.edits/0000000000000000001-wal.1733241423546.temp 2024-12-03T15:57:07,970 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:45015/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733241423446/wal.1733241423546; continuing. 2024-12-03T15:57:07,970 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:45015/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733241423446/wal.1733241423546 so closing down 2024-12-03T15:57:07,970 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-03T15:57:07,972 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-03T15:57:07,978 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44827 is added to blk_1073741856_1033 (size=497783) 2024-12-03T15:57:07,978 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46585 is added to blk_1073741856_1033 (size=497783) 2024-12-03T15:57:07,979 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:45015/hbase/data/default/testReplayEditsWrittenIntoWAL/0af130a07261dbe3a5b02b93d8b2bb99/recovered.edits/0000000000000000001-wal.1733241423546.temp (wrote 3002 edits, skipped 0 edits in 46 ms) 2024-12-03T15:57:07,983 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:45015/hbase/data/default/testReplayEditsWrittenIntoWAL/0af130a07261dbe3a5b02b93d8b2bb99/recovered.edits/0000000000000000001-wal.1733241423546.temp to hdfs://localhost:45015/hbase/data/default/testReplayEditsWrittenIntoWAL/0af130a07261dbe3a5b02b93d8b2bb99/recovered.edits/0000000000000003002 2024-12-03T15:57:07,983 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 3002 edits across 1 Regions in 146 ms; skipped=0; WAL=hdfs://localhost:45015/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733241423446/wal.1733241423546, size=0, length=0, corrupted=false, cancelled=false 2024-12-03T15:57:07,983 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:45015/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733241423446/wal.1733241423546, journal: Splitting hdfs://localhost:45015/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733241423446/wal.1733241423546, size=0 (0bytes) at 1733241423789Creating recovered edits writer path=hdfs://localhost:45015/hbase/data/default/testReplayEditsWrittenIntoWAL/0af130a07261dbe3a5b02b93d8b2bb99/recovered.edits/0000000000000000001-wal.1733241423546.temp at 1733241427904 (+4115 ms)Split 1024 edits, skipped 0 edits. at 1733241427929 (+25 ms)Split 2048 edits, skipped 0 edits. at 1733241427951 (+22 ms)Finishing writing output for hdfs://localhost:45015/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733241423446/wal.1733241423546 so closing down at 1733241427970 (+19 ms)3 split writer threads finished at 1733241427972 (+2 ms)Closed recovered edits writer path=hdfs://localhost:45015/hbase/data/default/testReplayEditsWrittenIntoWAL/0af130a07261dbe3a5b02b93d8b2bb99/recovered.edits/0000000000000000001-wal.1733241423546.temp (wrote 3002 edits, skipped 0 edits in 46 ms) at 1733241427979 (+7 ms)Rename recovered edits hdfs://localhost:45015/hbase/data/default/testReplayEditsWrittenIntoWAL/0af130a07261dbe3a5b02b93d8b2bb99/recovered.edits/0000000000000000001-wal.1733241423546.temp to hdfs://localhost:45015/hbase/data/default/testReplayEditsWrittenIntoWAL/0af130a07261dbe3a5b02b93d8b2bb99/recovered.edits/0000000000000003002 at 1733241427983 (+4 ms)Processed 3002 edits across 1 Regions in 146 ms; skipped=0; WAL=hdfs://localhost:45015/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733241423446/wal.1733241423546, size=0, length=0, corrupted=false, cancelled=false at 1733241427983 2024-12-03T15:57:07,986 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40575 is added to blk_1073741856_1033 (size=497783) 2024-12-03T15:57:07,987 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:45015/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733241423446/wal.1733241423546 to hdfs://localhost:45015/hbase/oldWALs/wal.1733241423546 2024-12-03T15:57:07,990 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:45015/hbase/data/default/testReplayEditsWrittenIntoWAL/0af130a07261dbe3a5b02b93d8b2bb99/recovered.edits/0000000000000003002 2024-12-03T15:57:07,990 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-03T15:57:07,996 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:45015/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733241423446, archiveDir=hdfs://localhost:45015/hbase/oldWALs, maxLogs=32 2024-12-03T15:57:08,017 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733241423446/wal.1733241427997, exclude list is [], retry=0 2024-12-03T15:57:08,021 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44827,DS-e007f1cc-fc1b-4368-9716-e11e4fe2f4eb,DISK] 2024-12-03T15:57:08,021 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40575,DS-309fe9b7-1997-44cb-b0c9-1ce3c04a71de,DISK] 2024-12-03T15:57:08,022 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46585,DS-5c6771fd-75c7-4775-ad88-ae5bbe88de5c,DISK] 2024-12-03T15:57:08,055 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733241423446/wal.1733241427997 2024-12-03T15:57:08,056 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46731:46731),(127.0.0.1/127.0.0.1:44513:44513),(127.0.0.1/127.0.0.1:37355:37355)] 2024-12-03T15:57:08,056 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1733241423448.0af130a07261dbe3a5b02b93d8b2bb99.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T15:57:08,059 INFO [StoreOpener-0af130a07261dbe3a5b02b93d8b2bb99-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 0af130a07261dbe3a5b02b93d8b2bb99 2024-12-03T15:57:08,061 INFO [StoreOpener-0af130a07261dbe3a5b02b93d8b2bb99-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0af130a07261dbe3a5b02b93d8b2bb99 columnFamilyName a 2024-12-03T15:57:08,061 DEBUG [StoreOpener-0af130a07261dbe3a5b02b93d8b2bb99-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T15:57:08,062 INFO [StoreOpener-0af130a07261dbe3a5b02b93d8b2bb99-1 {}] regionserver.HStore(327): Store=0af130a07261dbe3a5b02b93d8b2bb99/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T15:57:08,062 INFO [StoreOpener-0af130a07261dbe3a5b02b93d8b2bb99-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 0af130a07261dbe3a5b02b93d8b2bb99 2024-12-03T15:57:08,064 INFO [StoreOpener-0af130a07261dbe3a5b02b93d8b2bb99-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0af130a07261dbe3a5b02b93d8b2bb99 columnFamilyName b 2024-12-03T15:57:08,064 DEBUG [StoreOpener-0af130a07261dbe3a5b02b93d8b2bb99-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T15:57:08,065 INFO [StoreOpener-0af130a07261dbe3a5b02b93d8b2bb99-1 {}] regionserver.HStore(327): Store=0af130a07261dbe3a5b02b93d8b2bb99/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T15:57:08,065 INFO [StoreOpener-0af130a07261dbe3a5b02b93d8b2bb99-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 0af130a07261dbe3a5b02b93d8b2bb99 2024-12-03T15:57:08,066 INFO [StoreOpener-0af130a07261dbe3a5b02b93d8b2bb99-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0af130a07261dbe3a5b02b93d8b2bb99 columnFamilyName c 2024-12-03T15:57:08,066 DEBUG [StoreOpener-0af130a07261dbe3a5b02b93d8b2bb99-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T15:57:08,067 INFO [StoreOpener-0af130a07261dbe3a5b02b93d8b2bb99-1 {}] regionserver.HStore(327): Store=0af130a07261dbe3a5b02b93d8b2bb99/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T15:57:08,067 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 0af130a07261dbe3a5b02b93d8b2bb99 2024-12-03T15:57:08,078 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45015/hbase/data/default/testReplayEditsWrittenIntoWAL/0af130a07261dbe3a5b02b93d8b2bb99 2024-12-03T15:57:08,082 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:45015/hbase/data/default/testReplayEditsWrittenIntoWAL/0af130a07261dbe3a5b02b93d8b2bb99 2024-12-03T15:57:08,083 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:45015/hbase/data/default/testReplayEditsWrittenIntoWAL/0af130a07261dbe3a5b02b93d8b2bb99/recovered.edits/0000000000000003002 2024-12-03T15:57:08,129 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-03T15:57:08,184 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-12-03T15:57:08,690 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 0af130a07261dbe3a5b02b93d8b2bb99 3/3 column families, dataSize=42.49 KB heapSize=100.11 KB 2024-12-03T15:57:08,752 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45015/hbase/data/default/testReplayEditsWrittenIntoWAL/0af130a07261dbe3a5b02b93d8b2bb99/.tmp/a/8e97859e014f4963bfce732e45931330 is 62, key is testReplayEditsWrittenIntoWAL/a:100/1733241423600/Put/seqid=0 2024-12-03T15:57:08,774 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44827 is added to blk_1073741858_1035 (size=50463) 2024-12-03T15:57:08,774 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46585 is added to blk_1073741858_1035 (size=50463) 2024-12-03T15:57:08,775 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40575 is added to blk_1073741858_1035 (size=50463) 2024-12-03T15:57:08,776 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=754 (bloomFilter=true), to=hdfs://localhost:45015/hbase/data/default/testReplayEditsWrittenIntoWAL/0af130a07261dbe3a5b02b93d8b2bb99/.tmp/a/8e97859e014f4963bfce732e45931330 2024-12-03T15:57:08,788 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45015/hbase/data/default/testReplayEditsWrittenIntoWAL/0af130a07261dbe3a5b02b93d8b2bb99/.tmp/a/8e97859e014f4963bfce732e45931330 as hdfs://localhost:45015/hbase/data/default/testReplayEditsWrittenIntoWAL/0af130a07261dbe3a5b02b93d8b2bb99/a/8e97859e014f4963bfce732e45931330 2024-12-03T15:57:08,799 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45015/hbase/data/default/testReplayEditsWrittenIntoWAL/0af130a07261dbe3a5b02b93d8b2bb99/a/8e97859e014f4963bfce732e45931330, entries=754, sequenceid=754, filesize=49.3 K 2024-12-03T15:57:08,800 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~42.49 KB/43512, heapSize ~99.59 KB/101984, currentSize=0 B/0 for 0af130a07261dbe3a5b02b93d8b2bb99 in 110ms, sequenceid=754, compaction requested=false; wal=null 2024-12-03T15:57:08,821 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-12-03T15:57:08,821 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 0af130a07261dbe3a5b02b93d8b2bb99 3/3 column families, dataSize=42.49 KB heapSize=100.11 KB 2024-12-03T15:57:08,836 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45015/hbase/data/default/testReplayEditsWrittenIntoWAL/0af130a07261dbe3a5b02b93d8b2bb99/.tmp/a/262b27d950d145408f238628122ad8a5 is 62, key is testReplayEditsWrittenIntoWAL/a:754/1733241423635/Put/seqid=0 2024-12-03T15:57:08,858 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40575 is added to blk_1073741859_1036 (size=20072) 2024-12-03T15:57:08,859 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44827 is added to blk_1073741859_1036 (size=20072) 2024-12-03T15:57:08,859 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46585 is added to blk_1073741859_1036 (size=20072) 2024-12-03T15:57:08,862 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.93 KB at sequenceid=1508 (bloomFilter=true), to=hdfs://localhost:45015/hbase/data/default/testReplayEditsWrittenIntoWAL/0af130a07261dbe3a5b02b93d8b2bb99/.tmp/a/262b27d950d145408f238628122ad8a5 2024-12-03T15:57:08,903 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45015/hbase/data/default/testReplayEditsWrittenIntoWAL/0af130a07261dbe3a5b02b93d8b2bb99/.tmp/b/cdb602fd8bfb47188a7a58f0f9c78fc2 is 62, key is testReplayEditsWrittenIntoWAL/b:100/1733241423654/Put/seqid=0 2024-12-03T15:57:08,920 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40575 is added to blk_1073741860_1037 (size=35835) 2024-12-03T15:57:08,922 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44827 is added to blk_1073741860_1037 (size=35835) 2024-12-03T15:57:08,923 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46585 is added to blk_1073741860_1037 (size=35835) 2024-12-03T15:57:08,924 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=28.56 KB at sequenceid=1508 (bloomFilter=true), to=hdfs://localhost:45015/hbase/data/default/testReplayEditsWrittenIntoWAL/0af130a07261dbe3a5b02b93d8b2bb99/.tmp/b/cdb602fd8bfb47188a7a58f0f9c78fc2 2024-12-03T15:57:08,934 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45015/hbase/data/default/testReplayEditsWrittenIntoWAL/0af130a07261dbe3a5b02b93d8b2bb99/.tmp/a/262b27d950d145408f238628122ad8a5 as hdfs://localhost:45015/hbase/data/default/testReplayEditsWrittenIntoWAL/0af130a07261dbe3a5b02b93d8b2bb99/a/262b27d950d145408f238628122ad8a5 2024-12-03T15:57:08,946 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45015/hbase/data/default/testReplayEditsWrittenIntoWAL/0af130a07261dbe3a5b02b93d8b2bb99/a/262b27d950d145408f238628122ad8a5, entries=246, sequenceid=1508, filesize=19.6 K 2024-12-03T15:57:08,948 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45015/hbase/data/default/testReplayEditsWrittenIntoWAL/0af130a07261dbe3a5b02b93d8b2bb99/.tmp/b/cdb602fd8bfb47188a7a58f0f9c78fc2 as hdfs://localhost:45015/hbase/data/default/testReplayEditsWrittenIntoWAL/0af130a07261dbe3a5b02b93d8b2bb99/b/cdb602fd8bfb47188a7a58f0f9c78fc2 2024-12-03T15:57:08,958 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45015/hbase/data/default/testReplayEditsWrittenIntoWAL/0af130a07261dbe3a5b02b93d8b2bb99/b/cdb602fd8bfb47188a7a58f0f9c78fc2, entries=508, sequenceid=1508, filesize=35.0 K 2024-12-03T15:57:08,959 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~42.49 KB/43512, heapSize ~99.83 KB/102224, currentSize=0 B/0 for 0af130a07261dbe3a5b02b93d8b2bb99 in 138ms, sequenceid=1508, compaction requested=false; wal=null 2024-12-03T15:57:08,980 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-12-03T15:57:08,981 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 0af130a07261dbe3a5b02b93d8b2bb99 3/3 column families, dataSize=42.49 KB heapSize=100.11 KB 2024-12-03T15:57:08,990 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45015/hbase/data/default/testReplayEditsWrittenIntoWAL/0af130a07261dbe3a5b02b93d8b2bb99/.tmp/b/1c33eff836d2495c9bec63d80b6a73c6 is 62, key is testReplayEditsWrittenIntoWAL/b:508/1733241423673/Put/seqid=0 2024-12-03T15:57:09,011 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44827 is added to blk_1073741861_1038 (size=35082) 2024-12-03T15:57:09,011 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46585 is added to blk_1073741861_1038 (size=35082) 2024-12-03T15:57:09,012 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40575 is added to blk_1073741861_1038 (size=35082) 2024-12-03T15:57:09,013 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=27.87 KB at sequenceid=2262 (bloomFilter=true), to=hdfs://localhost:45015/hbase/data/default/testReplayEditsWrittenIntoWAL/0af130a07261dbe3a5b02b93d8b2bb99/.tmp/b/1c33eff836d2495c9bec63d80b6a73c6 2024-12-03T15:57:09,039 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45015/hbase/data/default/testReplayEditsWrittenIntoWAL/0af130a07261dbe3a5b02b93d8b2bb99/.tmp/c/9c3c8c4ba3b24224b92d5bdca8adfa9f is 62, key is testReplayEditsWrittenIntoWAL/c:100/1733241423707/Put/seqid=0 2024-12-03T15:57:09,055 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44827 is added to blk_1073741862_1039 (size=20825) 2024-12-03T15:57:09,055 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40575 is added to blk_1073741862_1039 (size=20825) 2024-12-03T15:57:09,055 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46585 is added to blk_1073741862_1039 (size=20825) 2024-12-03T15:57:09,057 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=14.63 KB at sequenceid=2262 (bloomFilter=true), to=hdfs://localhost:45015/hbase/data/default/testReplayEditsWrittenIntoWAL/0af130a07261dbe3a5b02b93d8b2bb99/.tmp/c/9c3c8c4ba3b24224b92d5bdca8adfa9f 2024-12-03T15:57:09,065 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45015/hbase/data/default/testReplayEditsWrittenIntoWAL/0af130a07261dbe3a5b02b93d8b2bb99/.tmp/b/1c33eff836d2495c9bec63d80b6a73c6 as hdfs://localhost:45015/hbase/data/default/testReplayEditsWrittenIntoWAL/0af130a07261dbe3a5b02b93d8b2bb99/b/1c33eff836d2495c9bec63d80b6a73c6 2024-12-03T15:57:09,072 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45015/hbase/data/default/testReplayEditsWrittenIntoWAL/0af130a07261dbe3a5b02b93d8b2bb99/b/1c33eff836d2495c9bec63d80b6a73c6, entries=492, sequenceid=2262, filesize=34.3 K 2024-12-03T15:57:09,074 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45015/hbase/data/default/testReplayEditsWrittenIntoWAL/0af130a07261dbe3a5b02b93d8b2bb99/.tmp/c/9c3c8c4ba3b24224b92d5bdca8adfa9f as hdfs://localhost:45015/hbase/data/default/testReplayEditsWrittenIntoWAL/0af130a07261dbe3a5b02b93d8b2bb99/c/9c3c8c4ba3b24224b92d5bdca8adfa9f 2024-12-03T15:57:09,082 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45015/hbase/data/default/testReplayEditsWrittenIntoWAL/0af130a07261dbe3a5b02b93d8b2bb99/c/9c3c8c4ba3b24224b92d5bdca8adfa9f, entries=262, sequenceid=2262, filesize=20.3 K 2024-12-03T15:57:09,083 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~42.49 KB/43512, heapSize ~99.83 KB/102224, currentSize=0 B/0 for 0af130a07261dbe3a5b02b93d8b2bb99 in 102ms, sequenceid=2262, compaction requested=false; wal=null 2024-12-03T15:57:09,093 WARN [Time-limited test {}] regionserver.HRegion(5722): No family for cell testReplayEditsWrittenIntoWAL/another family:testReplayEditsWrittenIntoWAL/1733241423753/Put/vlen=29/seqid=0 in region testReplayEditsWrittenIntoWAL,,1733241423448.0af130a07261dbe3a5b02b93d8b2bb99. 2024-12-03T15:57:09,098 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 3001, skipped 1, firstSequenceIdInLog=1, maxSequenceIdInLog=3002, path=hdfs://localhost:45015/hbase/data/default/testReplayEditsWrittenIntoWAL/0af130a07261dbe3a5b02b93d8b2bb99/recovered.edits/0000000000000003002 2024-12-03T15:57:09,098 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-12-03T15:57:09,098 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 0af130a07261dbe3a5b02b93d8b2bb99 3/3 column families, dataSize=41.85 KB heapSize=98.89 KB 2024-12-03T15:57:09,109 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45015/hbase/data/default/testReplayEditsWrittenIntoWAL/0af130a07261dbe3a5b02b93d8b2bb99/.tmp/c/303a04f917df46f6abd6f4377b80dbc6 is 62, key is testReplayEditsWrittenIntoWAL/c:262/1733241423713/Put/seqid=0 2024-12-03T15:57:09,126 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44827 is added to blk_1073741863_1040 (size=50301) 2024-12-03T15:57:09,126 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40575 is added to blk_1073741863_1040 (size=50301) 2024-12-03T15:57:09,126 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46585 is added to blk_1073741863_1040 (size=50301) 2024-12-03T15:57:09,127 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=41.85 KB at sequenceid=3002 (bloomFilter=true), to=hdfs://localhost:45015/hbase/data/default/testReplayEditsWrittenIntoWAL/0af130a07261dbe3a5b02b93d8b2bb99/.tmp/c/303a04f917df46f6abd6f4377b80dbc6 2024-12-03T15:57:09,139 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 303a04f917df46f6abd6f4377b80dbc6 2024-12-03T15:57:09,142 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45015/hbase/data/default/testReplayEditsWrittenIntoWAL/0af130a07261dbe3a5b02b93d8b2bb99/.tmp/c/303a04f917df46f6abd6f4377b80dbc6 as hdfs://localhost:45015/hbase/data/default/testReplayEditsWrittenIntoWAL/0af130a07261dbe3a5b02b93d8b2bb99/c/303a04f917df46f6abd6f4377b80dbc6 2024-12-03T15:57:09,155 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 303a04f917df46f6abd6f4377b80dbc6 2024-12-03T15:57:09,155 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45015/hbase/data/default/testReplayEditsWrittenIntoWAL/0af130a07261dbe3a5b02b93d8b2bb99/c/303a04f917df46f6abd6f4377b80dbc6, entries=739, sequenceid=3002, filesize=49.1 K 2024-12-03T15:57:09,156 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~41.85 KB/42854, heapSize ~98.38 KB/100736, currentSize=0 B/0 for 0af130a07261dbe3a5b02b93d8b2bb99 in 58ms, sequenceid=3002, compaction requested=false; wal=null 2024-12-03T15:57:09,157 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:45015/hbase/data/default/testReplayEditsWrittenIntoWAL/0af130a07261dbe3a5b02b93d8b2bb99/recovered.edits/0000000000000003002 2024-12-03T15:57:09,159 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 0af130a07261dbe3a5b02b93d8b2bb99 2024-12-03T15:57:09,159 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 0af130a07261dbe3a5b02b93d8b2bb99 2024-12-03T15:57:09,160 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenIntoWAL descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-03T15:57:09,162 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 0af130a07261dbe3a5b02b93d8b2bb99 2024-12-03T15:57:09,167 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45015/hbase/data/default/testReplayEditsWrittenIntoWAL/0af130a07261dbe3a5b02b93d8b2bb99/recovered.edits/3002.seqid, newMaxSeqId=3002, maxSeqId=1 2024-12-03T15:57:09,168 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 0af130a07261dbe3a5b02b93d8b2bb99; next sequenceid=3003; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=204800, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65960697, jitterRate=-0.017109021544456482}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-03T15:57:09,168 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 0af130a07261dbe3a5b02b93d8b2bb99: Writing region info on filesystem at 1733241428057Initializing all the Stores at 1733241428058 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733241428058Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733241428059 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733241428059Cleaning up temporary data from old regions at 1733241429159 (+1100 ms)Region opened successfully at 1733241429168 (+9 ms) 2024-12-03T15:57:09,235 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 0af130a07261dbe3a5b02b93d8b2bb99, disabling compactions & flushes 2024-12-03T15:57:09,235 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1733241423448.0af130a07261dbe3a5b02b93d8b2bb99. 2024-12-03T15:57:09,235 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1733241423448.0af130a07261dbe3a5b02b93d8b2bb99. 2024-12-03T15:57:09,235 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1733241423448.0af130a07261dbe3a5b02b93d8b2bb99. after waiting 0 ms 2024-12-03T15:57:09,235 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1733241423448.0af130a07261dbe3a5b02b93d8b2bb99. 2024-12-03T15:57:09,246 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1733241423448.0af130a07261dbe3a5b02b93d8b2bb99. 2024-12-03T15:57:09,246 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 0af130a07261dbe3a5b02b93d8b2bb99: Waiting for close lock at 1733241429234Disabling compacts and flushes for region at 1733241429234Disabling writes for close at 1733241429235 (+1 ms)Writing region close event to WAL at 1733241429246 (+11 ms)Closed at 1733241429246 2024-12-03T15:57:09,258 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40575 is added to blk_1073741857_1034 (size=175) 2024-12-03T15:57:09,258 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44827 is added to blk_1073741857_1034 (size=175) 2024-12-03T15:57:09,258 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46585 is added to blk_1073741857_1034 (size=175) 2024-12-03T15:57:09,264 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-03T15:57:09,265 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1733241427997) 2024-12-03T15:57:09,285 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsWrittenIntoWAL Thread=404 (was 388) Potentially hanging thread: IPC Client (1616464420) connection to localhost/127.0.0.1:35763 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1652044079_22 at /127.0.0.1:50782 [Waiting for operation #10] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:35763 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1652044079_22 at /127.0.0.1:53782 [Waiting for operation #6] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb3fe2df-6726-509d-596d-17caab0e5978/cluster_ef20cb83-4e1e-0871-5945-558338f0ff60/data/data1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb3fe2df-6726-509d-596d-17caab0e5978/cluster_ef20cb83-4e1e-0871-5945-558338f0ff60/data/data2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: IPC Client (1616464420) connection to localhost/127.0.0.1:34351 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:34351 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-12-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1652044079_22 at /127.0.0.1:40520 [Waiting for operation #20] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1616464420) connection to localhost/127.0.0.1:45015 from jenkins.replay.wal.secondtime java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: AsyncFSWAL-12-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb3fe2df-6726-509d-596d-17caab0e5978/cluster_ef20cb83-4e1e-0871-5945-558338f0ff60/data/data6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1652044079_22 at /127.0.0.1:53828 [Waiting for operation #12] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-12-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkins.replay.wal.secondtime@localhost:45015 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb3fe2df-6726-509d-596d-17caab0e5978/cluster_ef20cb83-4e1e-0871-5945-558338f0ff60/data/data5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=865 (was 785) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=920 (was 921), ProcessCount=11 (was 11), AvailableMemoryMB=5234 (was 5361) 2024-12-03T15:57:09,307 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#test2727 Thread=404, OpenFileDescriptor=865, MaxFileDescriptor=1048576, SystemLoadAverage=920, ProcessCount=11, AvailableMemoryMB=5230 2024-12-03T15:57:09,333 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-03T15:57:09,338 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-03T15:57:09,339 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-03T15:57:09,342 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-74854869, suffix=, logDir=hdfs://localhost:45015/hbase/WALs/hregion-74854869, archiveDir=hdfs://localhost:45015/hbase/oldWALs, maxLogs=32 2024-12-03T15:57:09,365 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-74854869/hregion-74854869.1733241429343, exclude list is [], retry=0 2024-12-03T15:57:09,370 DEBUG [AsyncFSWAL-14-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44827,DS-e007f1cc-fc1b-4368-9716-e11e4fe2f4eb,DISK] 2024-12-03T15:57:09,372 DEBUG [AsyncFSWAL-14-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40575,DS-309fe9b7-1997-44cb-b0c9-1ce3c04a71de,DISK] 2024-12-03T15:57:09,374 DEBUG [AsyncFSWAL-14-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46585,DS-5c6771fd-75c7-4775-ad88-ae5bbe88de5c,DISK] 2024-12-03T15:57:09,422 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-74854869/hregion-74854869.1733241429343 2024-12-03T15:57:09,425 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46731:46731),(127.0.0.1/127.0.0.1:44513:44513),(127.0.0.1/127.0.0.1:37355:37355)] 2024-12-03T15:57:09,425 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => e2994947b84d9c20a38cd1fc93abfaec, NAME => 'test2727,,1733241429334.e2994947b84d9c20a38cd1fc93abfaec.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='test2727', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45015/hbase 2024-12-03T15:57:09,446 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40575 is added to blk_1073741865_1042 (size=43) 2024-12-03T15:57:09,447 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44827 is added to blk_1073741865_1042 (size=43) 2024-12-03T15:57:09,447 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46585 is added to blk_1073741865_1042 (size=43) 2024-12-03T15:57:09,448 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated test2727,,1733241429334.e2994947b84d9c20a38cd1fc93abfaec.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T15:57:09,450 INFO [StoreOpener-e2994947b84d9c20a38cd1fc93abfaec-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region e2994947b84d9c20a38cd1fc93abfaec 2024-12-03T15:57:09,453 INFO [StoreOpener-e2994947b84d9c20a38cd1fc93abfaec-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e2994947b84d9c20a38cd1fc93abfaec columnFamilyName a 2024-12-03T15:57:09,453 DEBUG [StoreOpener-e2994947b84d9c20a38cd1fc93abfaec-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T15:57:09,453 INFO [StoreOpener-e2994947b84d9c20a38cd1fc93abfaec-1 {}] regionserver.HStore(327): Store=e2994947b84d9c20a38cd1fc93abfaec/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T15:57:09,454 INFO [StoreOpener-e2994947b84d9c20a38cd1fc93abfaec-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region e2994947b84d9c20a38cd1fc93abfaec 2024-12-03T15:57:09,456 INFO [StoreOpener-e2994947b84d9c20a38cd1fc93abfaec-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e2994947b84d9c20a38cd1fc93abfaec columnFamilyName b 2024-12-03T15:57:09,456 DEBUG [StoreOpener-e2994947b84d9c20a38cd1fc93abfaec-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T15:57:09,456 INFO [StoreOpener-e2994947b84d9c20a38cd1fc93abfaec-1 {}] regionserver.HStore(327): Store=e2994947b84d9c20a38cd1fc93abfaec/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T15:57:09,457 INFO [StoreOpener-e2994947b84d9c20a38cd1fc93abfaec-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region e2994947b84d9c20a38cd1fc93abfaec 2024-12-03T15:57:09,459 INFO [StoreOpener-e2994947b84d9c20a38cd1fc93abfaec-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e2994947b84d9c20a38cd1fc93abfaec columnFamilyName c 2024-12-03T15:57:09,459 DEBUG [StoreOpener-e2994947b84d9c20a38cd1fc93abfaec-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T15:57:09,459 INFO [StoreOpener-e2994947b84d9c20a38cd1fc93abfaec-1 {}] regionserver.HStore(327): Store=e2994947b84d9c20a38cd1fc93abfaec/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T15:57:09,460 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for e2994947b84d9c20a38cd1fc93abfaec 2024-12-03T15:57:09,466 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45015/hbase/data/default/test2727/e2994947b84d9c20a38cd1fc93abfaec 2024-12-03T15:57:09,467 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45015/hbase/data/default/test2727/e2994947b84d9c20a38cd1fc93abfaec 2024-12-03T15:57:09,469 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for e2994947b84d9c20a38cd1fc93abfaec 2024-12-03T15:57:09,469 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for e2994947b84d9c20a38cd1fc93abfaec 2024-12-03T15:57:09,470 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table test2727 descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-03T15:57:09,471 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for e2994947b84d9c20a38cd1fc93abfaec 2024-12-03T15:57:09,487 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45015/hbase/data/default/test2727/e2994947b84d9c20a38cd1fc93abfaec/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T15:57:09,488 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened e2994947b84d9c20a38cd1fc93abfaec; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71849135, jitterRate=0.07063554227352142}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-03T15:57:09,490 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for e2994947b84d9c20a38cd1fc93abfaec: Writing region info on filesystem at 1733241429448Initializing all the Stores at 1733241429450 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733241429450Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733241429450Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733241429450Cleaning up temporary data from old regions at 1733241429469 (+19 ms)Region opened successfully at 1733241429490 (+21 ms) 2024-12-03T15:57:09,491 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing e2994947b84d9c20a38cd1fc93abfaec, disabling compactions & flushes 2024-12-03T15:57:09,491 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region test2727,,1733241429334.e2994947b84d9c20a38cd1fc93abfaec. 2024-12-03T15:57:09,491 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on test2727,,1733241429334.e2994947b84d9c20a38cd1fc93abfaec. 2024-12-03T15:57:09,491 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on test2727,,1733241429334.e2994947b84d9c20a38cd1fc93abfaec. after waiting 0 ms 2024-12-03T15:57:09,491 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region test2727,,1733241429334.e2994947b84d9c20a38cd1fc93abfaec. 2024-12-03T15:57:09,492 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed test2727,,1733241429334.e2994947b84d9c20a38cd1fc93abfaec. 2024-12-03T15:57:09,492 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for e2994947b84d9c20a38cd1fc93abfaec: Waiting for close lock at 1733241429490Disabling compacts and flushes for region at 1733241429490Disabling writes for close at 1733241429491 (+1 ms)Writing region close event to WAL at 1733241429491Closed at 1733241429492 (+1 ms) 2024-12-03T15:57:09,511 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40575 is added to blk_1073741864_1041 (size=175) 2024-12-03T15:57:09,511 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44827 is added to blk_1073741864_1041 (size=175) 2024-12-03T15:57:09,519 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46585 is added to blk_1073741864_1041 (size=175) 2024-12-03T15:57:09,524 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-03T15:57:09,524 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-74854869:(num 1733241429343) 2024-12-03T15:57:09,524 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-03T15:57:09,527 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:45015/hbase/WALs/test2727-manual,16010,1733241429332, archiveDir=hdfs://localhost:45015/hbase/oldWALs, maxLogs=32 2024-12-03T15:57:09,546 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/test2727-manual,16010,1733241429332/wal.1733241429528, exclude list is [], retry=0 2024-12-03T15:57:09,550 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46585,DS-5c6771fd-75c7-4775-ad88-ae5bbe88de5c,DISK] 2024-12-03T15:57:09,550 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40575,DS-309fe9b7-1997-44cb-b0c9-1ce3c04a71de,DISK] 2024-12-03T15:57:09,551 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44827,DS-e007f1cc-fc1b-4368-9716-e11e4fe2f4eb,DISK] 2024-12-03T15:57:09,586 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/test2727-manual,16010,1733241429332/wal.1733241429528 2024-12-03T15:57:09,586 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37355:37355),(127.0.0.1/127.0.0.1:44513:44513),(127.0.0.1/127.0.0.1:46731:46731)] 2024-12-03T15:57:09,798 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44827 is added to blk_1073741866_1043 (size=371388) 2024-12-03T15:57:09,799 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46585 is added to blk_1073741866_1043 (size=371388) 2024-12-03T15:57:09,800 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40575 is added to blk_1073741866_1043 (size=371388) 2024-12-03T15:57:09,853 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:45015/hbase/WALs/test2727-manual,16010,1733241429332/wal.1733241429528, size=362.7 K (371388bytes) 2024-12-03T15:57:09,853 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:45015/hbase/WALs/test2727-manual,16010,1733241429332/wal.1733241429528 2024-12-03T15:57:09,853 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:45015/hbase/WALs/test2727-manual,16010,1733241429332/wal.1733241429528 after 0ms 2024-12-03T15:57:09,902 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-03T15:57:09,904 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:45015/hbase/WALs/test2727-manual,16010,1733241429332/wal.1733241429528 took 51ms 2024-12-03T15:57:09,918 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal.1733241429528.temp 2024-12-03T15:57:09,969 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:45015/hbase/data/default/test2727/e2994947b84d9c20a38cd1fc93abfaec/recovered.edits/0000000000000000001-wal.1733241429528.temp 2024-12-03T15:57:09,992 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:45015/hbase/WALs/test2727-manual,16010,1733241429332/wal.1733241429528 so closing down 2024-12-03T15:57:09,992 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-03T15:57:09,998 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-03T15:57:10,029 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40575 is added to blk_1073741867_1044 (size=371388) 2024-12-03T15:57:10,033 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:45015/hbase/data/default/test2727/e2994947b84d9c20a38cd1fc93abfaec/recovered.edits/0000000000000000001-wal.1733241429528.temp (wrote 3000 edits, skipped 0 edits in 28 ms) 2024-12-03T15:57:10,038 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44827 is added to blk_1073741867_1044 (size=371388) 2024-12-03T15:57:10,038 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46585 is added to blk_1073741867_1044 (size=371388) 2024-12-03T15:57:10,040 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:45015/hbase/data/default/test2727/e2994947b84d9c20a38cd1fc93abfaec/recovered.edits/0000000000000000001-wal.1733241429528.temp to hdfs://localhost:45015/hbase/data/default/test2727/e2994947b84d9c20a38cd1fc93abfaec/recovered.edits/0000000000000003000 2024-12-03T15:57:10,042 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 3000 edits across 1 Regions in 134 ms; skipped=0; WAL=hdfs://localhost:45015/hbase/WALs/test2727-manual,16010,1733241429332/wal.1733241429528, size=362.7 K, length=371388, corrupted=false, cancelled=false 2024-12-03T15:57:10,042 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:45015/hbase/WALs/test2727-manual,16010,1733241429332/wal.1733241429528, journal: Splitting hdfs://localhost:45015/hbase/WALs/test2727-manual,16010,1733241429332/wal.1733241429528, size=362.7 K (371388bytes) at 1733241429853Split 1024 edits, skipped 0 edits. at 1733241429936 (+83 ms)Split 2048 edits, skipped 0 edits. at 1733241429969 (+33 ms)Creating recovered edits writer path=hdfs://localhost:45015/hbase/data/default/test2727/e2994947b84d9c20a38cd1fc93abfaec/recovered.edits/0000000000000000001-wal.1733241429528.temp at 1733241429969Finishing writing output for hdfs://localhost:45015/hbase/WALs/test2727-manual,16010,1733241429332/wal.1733241429528 so closing down at 1733241429992 (+23 ms)3 split writer threads finished at 1733241429998 (+6 ms)Closed recovered edits writer path=hdfs://localhost:45015/hbase/data/default/test2727/e2994947b84d9c20a38cd1fc93abfaec/recovered.edits/0000000000000000001-wal.1733241429528.temp (wrote 3000 edits, skipped 0 edits in 28 ms) at 1733241430033 (+35 ms)Rename recovered edits hdfs://localhost:45015/hbase/data/default/test2727/e2994947b84d9c20a38cd1fc93abfaec/recovered.edits/0000000000000000001-wal.1733241429528.temp to hdfs://localhost:45015/hbase/data/default/test2727/e2994947b84d9c20a38cd1fc93abfaec/recovered.edits/0000000000000003000 at 1733241430040 (+7 ms)Processed 3000 edits across 1 Regions in 134 ms; skipped=0; WAL=hdfs://localhost:45015/hbase/WALs/test2727-manual,16010,1733241429332/wal.1733241429528, size=362.7 K, length=371388, corrupted=false, cancelled=false at 1733241430042 (+2 ms) 2024-12-03T15:57:10,048 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:45015/hbase/WALs/test2727-manual,16010,1733241429332/wal.1733241429528 to hdfs://localhost:45015/hbase/oldWALs/wal.1733241429528 2024-12-03T15:57:10,049 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:45015/hbase/data/default/test2727/e2994947b84d9c20a38cd1fc93abfaec/recovered.edits/0000000000000003000 2024-12-03T15:57:10,049 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-03T15:57:10,053 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:45015/hbase/WALs/test2727-manual,16010,1733241429332, archiveDir=hdfs://localhost:45015/hbase/oldWALs, maxLogs=32 2024-12-03T15:57:10,076 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/test2727-manual,16010,1733241429332/wal.1733241430054, exclude list is [], retry=0 2024-12-03T15:57:10,095 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44827,DS-e007f1cc-fc1b-4368-9716-e11e4fe2f4eb,DISK] 2024-12-03T15:57:10,096 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40575,DS-309fe9b7-1997-44cb-b0c9-1ce3c04a71de,DISK] 2024-12-03T15:57:10,097 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46585,DS-5c6771fd-75c7-4775-ad88-ae5bbe88de5c,DISK] 2024-12-03T15:57:10,155 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/test2727-manual,16010,1733241429332/wal.1733241430054 2024-12-03T15:57:10,156 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46731:46731),(127.0.0.1/127.0.0.1:44513:44513),(127.0.0.1/127.0.0.1:37355:37355)] 2024-12-03T15:57:10,364 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44827 is added to blk_1073741868_1045 (size=371515) 2024-12-03T15:57:10,365 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40575 is added to blk_1073741868_1045 (size=371515) 2024-12-03T15:57:10,365 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46585 is added to blk_1073741868_1045 (size=371515) 2024-12-03T15:57:10,395 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:45015/hbase/WALs/test2727-manual,16010,1733241429332/wal.1733241430054, size=362.8 K (371515bytes) 2024-12-03T15:57:10,395 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:45015/hbase/WALs/test2727-manual,16010,1733241429332/wal.1733241430054 2024-12-03T15:57:10,396 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:45015/hbase/WALs/test2727-manual,16010,1733241429332/wal.1733241430054 after 1ms 2024-12-03T15:57:10,445 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-03T15:57:10,448 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:45015/hbase/WALs/test2727-manual,16010,1733241429332/wal.1733241430054 took 53ms 2024-12-03T15:57:10,506 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000003001-wal.1733241430054.temp 2024-12-03T15:57:10,525 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:45015/hbase/WALs/test2727-manual,16010,1733241429332/wal.1733241430054 so closing down 2024-12-03T15:57:10,525 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-03T15:57:10,571 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:45015/hbase/data/default/test2727/e2994947b84d9c20a38cd1fc93abfaec/recovered.edits/0000000000000003001-wal.1733241430054.temp 2024-12-03T15:57:10,587 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-03T15:57:10,603 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44827 is added to blk_1073741869_1046 (size=371515) 2024-12-03T15:57:10,603 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40575 is added to blk_1073741869_1046 (size=371515) 2024-12-03T15:57:10,607 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46585 is added to blk_1073741869_1046 (size=371515) 2024-12-03T15:57:11,003 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:45015/hbase/data/default/test2727/e2994947b84d9c20a38cd1fc93abfaec/recovered.edits/0000000000000003001-wal.1733241430054.temp (wrote 3000 edits, skipped 0 edits in 15 ms) 2024-12-03T15:57:11,005 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:45015/hbase/data/default/test2727/e2994947b84d9c20a38cd1fc93abfaec/recovered.edits/0000000000000003001-wal.1733241430054.temp to hdfs://localhost:45015/hbase/data/default/test2727/e2994947b84d9c20a38cd1fc93abfaec/recovered.edits/0000000000000006000 2024-12-03T15:57:11,005 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 3000 edits across 1 Regions in 555 ms; skipped=0; WAL=hdfs://localhost:45015/hbase/WALs/test2727-manual,16010,1733241429332/wal.1733241430054, size=362.8 K, length=371515, corrupted=false, cancelled=false 2024-12-03T15:57:11,006 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:45015/hbase/WALs/test2727-manual,16010,1733241429332/wal.1733241430054, journal: Splitting hdfs://localhost:45015/hbase/WALs/test2727-manual,16010,1733241429332/wal.1733241430054, size=362.8 K (371515bytes) at 1733241430395Split 1024 edits, skipped 0 edits. at 1733241430506 (+111 ms)Split 2048 edits, skipped 0 edits. at 1733241430517 (+11 ms)Finishing writing output for hdfs://localhost:45015/hbase/WALs/test2727-manual,16010,1733241429332/wal.1733241430054 so closing down at 1733241430525 (+8 ms)Creating recovered edits writer path=hdfs://localhost:45015/hbase/data/default/test2727/e2994947b84d9c20a38cd1fc93abfaec/recovered.edits/0000000000000003001-wal.1733241430054.temp at 1733241430571 (+46 ms)3 split writer threads finished at 1733241430587 (+16 ms)Closed recovered edits writer path=hdfs://localhost:45015/hbase/data/default/test2727/e2994947b84d9c20a38cd1fc93abfaec/recovered.edits/0000000000000003001-wal.1733241430054.temp (wrote 3000 edits, skipped 0 edits in 15 ms) at 1733241431003 (+416 ms)Rename recovered edits hdfs://localhost:45015/hbase/data/default/test2727/e2994947b84d9c20a38cd1fc93abfaec/recovered.edits/0000000000000003001-wal.1733241430054.temp to hdfs://localhost:45015/hbase/data/default/test2727/e2994947b84d9c20a38cd1fc93abfaec/recovered.edits/0000000000000006000 at 1733241431005 (+2 ms)Processed 3000 edits across 1 Regions in 555 ms; skipped=0; WAL=hdfs://localhost:45015/hbase/WALs/test2727-manual,16010,1733241429332/wal.1733241430054, size=362.8 K, length=371515, corrupted=false, cancelled=false at 1733241431005 2024-12-03T15:57:11,008 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:45015/hbase/WALs/test2727-manual,16010,1733241429332/wal.1733241430054 to hdfs://localhost:45015/hbase/oldWALs/wal.1733241430054 2024-12-03T15:57:11,010 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:45015/hbase/data/default/test2727/e2994947b84d9c20a38cd1fc93abfaec/recovered.edits/0000000000000006000 2024-12-03T15:57:11,010 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-03T15:57:11,013 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:45015/hbase/WALs/test2727-manual,16010,1733241429332, archiveDir=hdfs://localhost:45015/hbase/oldWALs, maxLogs=32 2024-12-03T15:57:11,035 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/test2727-manual,16010,1733241429332/wal.1733241431013, exclude list is [], retry=0 2024-12-03T15:57:11,039 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40575,DS-309fe9b7-1997-44cb-b0c9-1ce3c04a71de,DISK] 2024-12-03T15:57:11,040 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44827,DS-e007f1cc-fc1b-4368-9716-e11e4fe2f4eb,DISK] 2024-12-03T15:57:11,040 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46585,DS-5c6771fd-75c7-4775-ad88-ae5bbe88de5c,DISK] 2024-12-03T15:57:11,085 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/test2727-manual,16010,1733241429332/wal.1733241431013 2024-12-03T15:57:11,085 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44513:44513),(127.0.0.1/127.0.0.1:46731:46731),(127.0.0.1/127.0.0.1:37355:37355)] 2024-12-03T15:57:11,085 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => e2994947b84d9c20a38cd1fc93abfaec, NAME => 'test2727,,1733241429334.e2994947b84d9c20a38cd1fc93abfaec.', STARTKEY => '', ENDKEY => ''} 2024-12-03T15:57:11,086 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated test2727,,1733241429334.e2994947b84d9c20a38cd1fc93abfaec.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T15:57:11,086 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for e2994947b84d9c20a38cd1fc93abfaec 2024-12-03T15:57:11,086 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for e2994947b84d9c20a38cd1fc93abfaec 2024-12-03T15:57:11,088 INFO [StoreOpener-e2994947b84d9c20a38cd1fc93abfaec-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region e2994947b84d9c20a38cd1fc93abfaec 2024-12-03T15:57:11,090 INFO [StoreOpener-e2994947b84d9c20a38cd1fc93abfaec-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e2994947b84d9c20a38cd1fc93abfaec columnFamilyName a 2024-12-03T15:57:11,091 DEBUG [StoreOpener-e2994947b84d9c20a38cd1fc93abfaec-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T15:57:11,092 INFO [StoreOpener-e2994947b84d9c20a38cd1fc93abfaec-1 {}] regionserver.HStore(327): Store=e2994947b84d9c20a38cd1fc93abfaec/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T15:57:11,092 INFO [StoreOpener-e2994947b84d9c20a38cd1fc93abfaec-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region e2994947b84d9c20a38cd1fc93abfaec 2024-12-03T15:57:11,094 INFO [StoreOpener-e2994947b84d9c20a38cd1fc93abfaec-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e2994947b84d9c20a38cd1fc93abfaec columnFamilyName b 2024-12-03T15:57:11,094 DEBUG [StoreOpener-e2994947b84d9c20a38cd1fc93abfaec-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T15:57:11,097 INFO [StoreOpener-e2994947b84d9c20a38cd1fc93abfaec-1 {}] regionserver.HStore(327): Store=e2994947b84d9c20a38cd1fc93abfaec/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T15:57:11,097 INFO [StoreOpener-e2994947b84d9c20a38cd1fc93abfaec-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region e2994947b84d9c20a38cd1fc93abfaec 2024-12-03T15:57:11,098 INFO [StoreOpener-e2994947b84d9c20a38cd1fc93abfaec-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e2994947b84d9c20a38cd1fc93abfaec columnFamilyName c 2024-12-03T15:57:11,099 DEBUG [StoreOpener-e2994947b84d9c20a38cd1fc93abfaec-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T15:57:11,099 INFO [StoreOpener-e2994947b84d9c20a38cd1fc93abfaec-1 {}] regionserver.HStore(327): Store=e2994947b84d9c20a38cd1fc93abfaec/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T15:57:11,099 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for e2994947b84d9c20a38cd1fc93abfaec 2024-12-03T15:57:11,100 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45015/hbase/data/default/test2727/e2994947b84d9c20a38cd1fc93abfaec 2024-12-03T15:57:11,106 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 2 recovered edits file(s) under hdfs://localhost:45015/hbase/data/default/test2727/e2994947b84d9c20a38cd1fc93abfaec 2024-12-03T15:57:11,107 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:45015/hbase/data/default/test2727/e2994947b84d9c20a38cd1fc93abfaec/recovered.edits/0000000000000003000 2024-12-03T15:57:11,144 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-03T15:57:11,175 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 3000, skipped 0, firstSequenceIdInLog=1, maxSequenceIdInLog=3000, path=hdfs://localhost:45015/hbase/data/default/test2727/e2994947b84d9c20a38cd1fc93abfaec/recovered.edits/0000000000000003000 2024-12-03T15:57:11,177 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:45015/hbase/data/default/test2727/e2994947b84d9c20a38cd1fc93abfaec/recovered.edits/0000000000000006000 2024-12-03T15:57:11,212 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-03T15:57:11,242 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 3000, skipped 0, firstSequenceIdInLog=3001, maxSequenceIdInLog=6000, path=hdfs://localhost:45015/hbase/data/default/test2727/e2994947b84d9c20a38cd1fc93abfaec/recovered.edits/0000000000000006000 2024-12-03T15:57:11,242 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing e2994947b84d9c20a38cd1fc93abfaec 3/3 column families, dataSize=215.51 KB heapSize=657 KB 2024-12-03T15:57:11,269 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45015/hbase/data/default/test2727/e2994947b84d9c20a38cd1fc93abfaec/.tmp/a/3537c29b58b843e2869c27489ab656ff is 41, key is test2727/a:100/1733241430162/Put/seqid=0 2024-12-03T15:57:11,283 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46585 is added to blk_1073741871_1048 (size=84227) 2024-12-03T15:57:11,283 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44827 is added to blk_1073741871_1048 (size=84227) 2024-12-03T15:57:11,284 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40575 is added to blk_1073741871_1048 (size=84227) 2024-12-03T15:57:11,285 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=71.84 KB at sequenceid=6000 (bloomFilter=true), to=hdfs://localhost:45015/hbase/data/default/test2727/e2994947b84d9c20a38cd1fc93abfaec/.tmp/a/3537c29b58b843e2869c27489ab656ff 2024-12-03T15:57:11,313 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45015/hbase/data/default/test2727/e2994947b84d9c20a38cd1fc93abfaec/.tmp/b/c79f8267e2804eb793897cb2139e94a0 is 41, key is test2727/b:100/1733241430239/Put/seqid=0 2024-12-03T15:57:11,327 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40575 is added to blk_1073741872_1049 (size=84609) 2024-12-03T15:57:11,327 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46585 is added to blk_1073741872_1049 (size=84609) 2024-12-03T15:57:11,328 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44827 is added to blk_1073741872_1049 (size=84609) 2024-12-03T15:57:11,341 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=71.84 KB at sequenceid=6000 (bloomFilter=true), to=hdfs://localhost:45015/hbase/data/default/test2727/e2994947b84d9c20a38cd1fc93abfaec/.tmp/b/c79f8267e2804eb793897cb2139e94a0 2024-12-03T15:57:11,389 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45015/hbase/data/default/test2727/e2994947b84d9c20a38cd1fc93abfaec/.tmp/c/5d4031ef1c904119bffe25d7b9df49dc is 41, key is test2727/c:100/1733241430307/Put/seqid=0 2024-12-03T15:57:11,408 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46585 is added to blk_1073741873_1050 (size=84609) 2024-12-03T15:57:11,408 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44827 is added to blk_1073741873_1050 (size=84609) 2024-12-03T15:57:11,408 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40575 is added to blk_1073741873_1050 (size=84609) 2024-12-03T15:57:11,809 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=71.84 KB at sequenceid=6000 (bloomFilter=true), to=hdfs://localhost:45015/hbase/data/default/test2727/e2994947b84d9c20a38cd1fc93abfaec/.tmp/c/5d4031ef1c904119bffe25d7b9df49dc 2024-12-03T15:57:11,817 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45015/hbase/data/default/test2727/e2994947b84d9c20a38cd1fc93abfaec/.tmp/a/3537c29b58b843e2869c27489ab656ff as hdfs://localhost:45015/hbase/data/default/test2727/e2994947b84d9c20a38cd1fc93abfaec/a/3537c29b58b843e2869c27489ab656ff 2024-12-03T15:57:11,824 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45015/hbase/data/default/test2727/e2994947b84d9c20a38cd1fc93abfaec/a/3537c29b58b843e2869c27489ab656ff, entries=2000, sequenceid=6000, filesize=82.3 K 2024-12-03T15:57:11,825 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45015/hbase/data/default/test2727/e2994947b84d9c20a38cd1fc93abfaec/.tmp/b/c79f8267e2804eb793897cb2139e94a0 as hdfs://localhost:45015/hbase/data/default/test2727/e2994947b84d9c20a38cd1fc93abfaec/b/c79f8267e2804eb793897cb2139e94a0 2024-12-03T15:57:11,833 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45015/hbase/data/default/test2727/e2994947b84d9c20a38cd1fc93abfaec/b/c79f8267e2804eb793897cb2139e94a0, entries=2000, sequenceid=6000, filesize=82.6 K 2024-12-03T15:57:11,834 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45015/hbase/data/default/test2727/e2994947b84d9c20a38cd1fc93abfaec/.tmp/c/5d4031ef1c904119bffe25d7b9df49dc as hdfs://localhost:45015/hbase/data/default/test2727/e2994947b84d9c20a38cd1fc93abfaec/c/5d4031ef1c904119bffe25d7b9df49dc 2024-12-03T15:57:11,841 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45015/hbase/data/default/test2727/e2994947b84d9c20a38cd1fc93abfaec/c/5d4031ef1c904119bffe25d7b9df49dc, entries=2000, sequenceid=6000, filesize=82.6 K 2024-12-03T15:57:11,842 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~215.51 KB/220680, heapSize ~656.95 KB/672720, currentSize=0 B/0 for e2994947b84d9c20a38cd1fc93abfaec in 600ms, sequenceid=6000, compaction requested=false; wal=null 2024-12-03T15:57:11,843 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:45015/hbase/data/default/test2727/e2994947b84d9c20a38cd1fc93abfaec/recovered.edits/0000000000000003000 2024-12-03T15:57:11,844 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:45015/hbase/data/default/test2727/e2994947b84d9c20a38cd1fc93abfaec/recovered.edits/0000000000000006000 2024-12-03T15:57:11,845 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for e2994947b84d9c20a38cd1fc93abfaec 2024-12-03T15:57:11,845 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for e2994947b84d9c20a38cd1fc93abfaec 2024-12-03T15:57:11,845 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table test2727 descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-03T15:57:11,847 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for e2994947b84d9c20a38cd1fc93abfaec 2024-12-03T15:57:11,849 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45015/hbase/data/default/test2727/e2994947b84d9c20a38cd1fc93abfaec/recovered.edits/6000.seqid, newMaxSeqId=6000, maxSeqId=1 2024-12-03T15:57:11,851 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened e2994947b84d9c20a38cd1fc93abfaec; next sequenceid=6001; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61596274, jitterRate=-0.08214399218559265}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-03T15:57:11,851 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for e2994947b84d9c20a38cd1fc93abfaec: Writing region info on filesystem at 1733241431086Initializing all the Stores at 1733241431088 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733241431088Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733241431088Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733241431088Obtaining lock to block concurrent updates at 1733241431242 (+154 ms)Preparing flush snapshotting stores in e2994947b84d9c20a38cd1fc93abfaec at 1733241431242Finished memstore snapshotting test2727,,1733241429334.e2994947b84d9c20a38cd1fc93abfaec., syncing WAL and waiting on mvcc, flushsize=dataSize=220680, getHeapSize=672720, getOffHeapSize=0, getCellsCount=6000 at 1733241431242Flushing stores of test2727,,1733241429334.e2994947b84d9c20a38cd1fc93abfaec. at 1733241431242Flushing e2994947b84d9c20a38cd1fc93abfaec/a: creating writer at 1733241431243 (+1 ms)Flushing e2994947b84d9c20a38cd1fc93abfaec/a: appending metadata at 1733241431268 (+25 ms)Flushing e2994947b84d9c20a38cd1fc93abfaec/a: closing flushed file at 1733241431268Flushing e2994947b84d9c20a38cd1fc93abfaec/b: creating writer at 1733241431292 (+24 ms)Flushing e2994947b84d9c20a38cd1fc93abfaec/b: appending metadata at 1733241431311 (+19 ms)Flushing e2994947b84d9c20a38cd1fc93abfaec/b: closing flushed file at 1733241431311Flushing e2994947b84d9c20a38cd1fc93abfaec/c: creating writer at 1733241431350 (+39 ms)Flushing e2994947b84d9c20a38cd1fc93abfaec/c: appending metadata at 1733241431388 (+38 ms)Flushing e2994947b84d9c20a38cd1fc93abfaec/c: closing flushed file at 1733241431388Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2f893a37: reopening flushed file at 1733241431816 (+428 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3cfd0599: reopening flushed file at 1733241431824 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@465b5ec9: reopening flushed file at 1733241431833 (+9 ms)Finished flush of dataSize ~215.51 KB/220680, heapSize ~656.95 KB/672720, currentSize=0 B/0 for e2994947b84d9c20a38cd1fc93abfaec in 600ms, sequenceid=6000, compaction requested=false; wal=null at 1733241431842 (+9 ms)Cleaning up temporary data from old regions at 1733241431845 (+3 ms)Region opened successfully at 1733241431851 (+6 ms) 2024-12-03T15:57:11,852 DEBUG [Time-limited test {}] wal.AbstractTestWALReplay(320): region.getOpenSeqNum(): 6001, wal3.id: 0 2024-12-03T15:57:11,852 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing e2994947b84d9c20a38cd1fc93abfaec, disabling compactions & flushes 2024-12-03T15:57:11,852 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region test2727,,1733241429334.e2994947b84d9c20a38cd1fc93abfaec. 2024-12-03T15:57:11,853 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on test2727,,1733241429334.e2994947b84d9c20a38cd1fc93abfaec. 2024-12-03T15:57:11,853 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on test2727,,1733241429334.e2994947b84d9c20a38cd1fc93abfaec. after waiting 0 ms 2024-12-03T15:57:11,853 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region test2727,,1733241429334.e2994947b84d9c20a38cd1fc93abfaec. 2024-12-03T15:57:11,854 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed test2727,,1733241429334.e2994947b84d9c20a38cd1fc93abfaec. 2024-12-03T15:57:11,854 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for e2994947b84d9c20a38cd1fc93abfaec: Waiting for close lock at 1733241431852Disabling compacts and flushes for region at 1733241431852Disabling writes for close at 1733241431853 (+1 ms)Writing region close event to WAL at 1733241431854 (+1 ms)Closed at 1733241431854 2024-12-03T15:57:11,857 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40575 is added to blk_1073741870_1047 (size=175) 2024-12-03T15:57:11,857 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46585 is added to blk_1073741870_1047 (size=175) 2024-12-03T15:57:11,858 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44827 is added to blk_1073741870_1047 (size=175) 2024-12-03T15:57:11,861 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-03T15:57:11,861 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1733241431013) 2024-12-03T15:57:11,877 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#test2727 Thread=401 (was 404), OpenFileDescriptor=921 (was 865) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=886 (was 920), ProcessCount=11 (was 11), AvailableMemoryMB=5015 (was 5230) 2024-12-03T15:57:11,891 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testSequentialEditLogSeqNum Thread=401, OpenFileDescriptor=921, MaxFileDescriptor=1048576, SystemLoadAverage=886, ProcessCount=11, AvailableMemoryMB=5014 2024-12-03T15:57:11,914 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-03T15:57:11,923 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:45015/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733241431913, archiveDir=hdfs://localhost:45015/hbase/oldWALs, maxLogs=32 2024-12-03T15:57:11,923 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733241431923 2024-12-03T15:57:11,972 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733241431913/wal.1733241431923 2024-12-03T15:57:11,977 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new MockWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44513:44513),(127.0.0.1/127.0.0.1:46731:46731),(127.0.0.1/127.0.0.1:37355:37355)] 2024-12-03T15:57:11,981 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 220731e525589e0a967d40a4d4b18006, NAME => 'testSequentialEditLogSeqNum,,1733241431915.220731e525589e0a967d40a4d4b18006.', STARTKEY => '', ENDKEY => ''} 2024-12-03T15:57:11,981 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testSequentialEditLogSeqNum,,1733241431915.220731e525589e0a967d40a4d4b18006.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T15:57:11,981 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 220731e525589e0a967d40a4d4b18006 2024-12-03T15:57:11,981 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 220731e525589e0a967d40a4d4b18006 2024-12-03T15:57:11,983 WARN [Time-limited test {}] regionserver.HRegionFileSystem(836): hdfs://localhost:45015/hbase/data/default/testSequentialEditLogSeqNum/220731e525589e0a967d40a4d4b18006 doesn't exist for region: 220731e525589e0a967d40a4d4b18006 on table testSequentialEditLogSeqNum 2024-12-03T15:57:11,983 WARN [Time-limited test {}] regionserver.HRegionFileSystem(854): .regioninfo file not found for region: 220731e525589e0a967d40a4d4b18006 on table testSequentialEditLogSeqNum 2024-12-03T15:57:11,997 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40575 is added to blk_1073741875_1052 (size=62) 2024-12-03T15:57:11,997 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46585 is added to blk_1073741875_1052 (size=62) 2024-12-03T15:57:11,998 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44827 is added to blk_1073741875_1052 (size=62) 2024-12-03T15:57:12,003 INFO [StoreOpener-220731e525589e0a967d40a4d4b18006-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 220731e525589e0a967d40a4d4b18006 2024-12-03T15:57:12,005 INFO [StoreOpener-220731e525589e0a967d40a4d4b18006-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 220731e525589e0a967d40a4d4b18006 columnFamilyName a 2024-12-03T15:57:12,006 DEBUG [StoreOpener-220731e525589e0a967d40a4d4b18006-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T15:57:12,006 INFO [StoreOpener-220731e525589e0a967d40a4d4b18006-1 {}] regionserver.HStore(327): Store=220731e525589e0a967d40a4d4b18006/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T15:57:12,007 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 220731e525589e0a967d40a4d4b18006 2024-12-03T15:57:12,007 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45015/hbase/data/default/testSequentialEditLogSeqNum/220731e525589e0a967d40a4d4b18006 2024-12-03T15:57:12,008 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45015/hbase/data/default/testSequentialEditLogSeqNum/220731e525589e0a967d40a4d4b18006 2024-12-03T15:57:12,009 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 220731e525589e0a967d40a4d4b18006 2024-12-03T15:57:12,009 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 220731e525589e0a967d40a4d4b18006 2024-12-03T15:57:12,011 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 220731e525589e0a967d40a4d4b18006 2024-12-03T15:57:12,014 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45015/hbase/data/default/testSequentialEditLogSeqNum/220731e525589e0a967d40a4d4b18006/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T15:57:12,015 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 220731e525589e0a967d40a4d4b18006; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75350950, jitterRate=0.12281665205955505}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-03T15:57:12,016 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 220731e525589e0a967d40a4d4b18006: Writing region info on filesystem at 1733241431981Initializing all the Stores at 1733241432003 (+22 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733241432003Cleaning up temporary data from old regions at 1733241432009 (+6 ms)Region opened successfully at 1733241432016 (+7 ms) 2024-12-03T15:57:12,031 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 220731e525589e0a967d40a4d4b18006 1/1 column families, dataSize=770 B heapSize=1.73 KB 2024-12-03T15:57:12,061 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45015/hbase/data/default/testSequentialEditLogSeqNum/220731e525589e0a967d40a4d4b18006/.tmp/a/96e9e7bfb0e441f7aa52d783276957cc is 81, key is testSequentialEditLogSeqNum/a:x0/1733241432016/Put/seqid=0 2024-12-03T15:57:12,080 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46585 is added to blk_1073741876_1053 (size=5833) 2024-12-03T15:57:12,081 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40575 is added to blk_1073741876_1053 (size=5833) 2024-12-03T15:57:12,081 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44827 is added to blk_1073741876_1053 (size=5833) 2024-12-03T15:57:12,087 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=770 B at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:45015/hbase/data/default/testSequentialEditLogSeqNum/220731e525589e0a967d40a4d4b18006/.tmp/a/96e9e7bfb0e441f7aa52d783276957cc 2024-12-03T15:57:12,097 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45015/hbase/data/default/testSequentialEditLogSeqNum/220731e525589e0a967d40a4d4b18006/.tmp/a/96e9e7bfb0e441f7aa52d783276957cc as hdfs://localhost:45015/hbase/data/default/testSequentialEditLogSeqNum/220731e525589e0a967d40a4d4b18006/a/96e9e7bfb0e441f7aa52d783276957cc 2024-12-03T15:57:12,106 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45015/hbase/data/default/testSequentialEditLogSeqNum/220731e525589e0a967d40a4d4b18006/a/96e9e7bfb0e441f7aa52d783276957cc, entries=10, sequenceid=13, filesize=5.7 K 2024-12-03T15:57:12,108 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~770 B/770, heapSize ~1.72 KB/1760, currentSize=0 B/0 for 220731e525589e0a967d40a4d4b18006 in 77ms, sequenceid=13, compaction requested=false 2024-12-03T15:57:12,108 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 220731e525589e0a967d40a4d4b18006: 2024-12-03T15:57:12,121 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-03T15:57:12,121 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-03T15:57:12,121 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-03T15:57:12,121 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-03T15:57:12,121 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-03T15:57:12,124 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46585 is added to blk_1073741874_1051 (size=3256) 2024-12-03T15:57:12,125 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44827 is added to blk_1073741874_1051 (size=3256) 2024-12-03T15:57:12,125 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40575 is added to blk_1073741874_1051 (size=3256) 2024-12-03T15:57:12,144 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:45015/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733241431913/wal.1733241431923, size=3.2 K (3256bytes) 2024-12-03T15:57:12,144 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:45015/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733241431913/wal.1733241431923 2024-12-03T15:57:12,145 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:45015/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733241431913/wal.1733241431923 after 1ms 2024-12-03T15:57:12,180 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-03T15:57:12,180 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:45015/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733241431913/wal.1733241431923 took 36ms 2024-12-03T15:57:12,184 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:45015/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733241431913/wal.1733241431923 so closing down 2024-12-03T15:57:12,185 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-03T15:57:12,186 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1733241431923.temp 2024-12-03T15:57:12,218 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:45015/hbase/data/default/testSequentialEditLogSeqNum/220731e525589e0a967d40a4d4b18006/recovered.edits/0000000000000000003-wal.1733241431923.temp 2024-12-03T15:57:12,218 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-03T15:57:12,226 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44827 is added to blk_1073741877_1054 (size=2650) 2024-12-03T15:57:12,226 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40575 is added to blk_1073741877_1054 (size=2650) 2024-12-03T15:57:12,229 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46585 is added to blk_1073741877_1054 (size=2650) 2024-12-03T15:57:12,230 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:45015/hbase/data/default/testSequentialEditLogSeqNum/220731e525589e0a967d40a4d4b18006/recovered.edits/0000000000000000003-wal.1733241431923.temp (wrote 15 edits, skipped 0 edits in 0 ms) 2024-12-03T15:57:12,231 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:45015/hbase/data/default/testSequentialEditLogSeqNum/220731e525589e0a967d40a4d4b18006/recovered.edits/0000000000000000003-wal.1733241431923.temp to hdfs://localhost:45015/hbase/data/default/testSequentialEditLogSeqNum/220731e525589e0a967d40a4d4b18006/recovered.edits/0000000000000000020 2024-12-03T15:57:12,234 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 17 edits across 1 Regions in 52 ms; skipped=2; WAL=hdfs://localhost:45015/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733241431913/wal.1733241431923, size=3.2 K, length=3256, corrupted=false, cancelled=false 2024-12-03T15:57:12,234 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:45015/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733241431913/wal.1733241431923, journal: Splitting hdfs://localhost:45015/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733241431913/wal.1733241431923, size=3.2 K (3256bytes) at 1733241432144Finishing writing output for hdfs://localhost:45015/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733241431913/wal.1733241431923 so closing down at 1733241432185 (+41 ms)Creating recovered edits writer path=hdfs://localhost:45015/hbase/data/default/testSequentialEditLogSeqNum/220731e525589e0a967d40a4d4b18006/recovered.edits/0000000000000000003-wal.1733241431923.temp at 1733241432218 (+33 ms)3 split writer threads finished at 1733241432218Closed recovered edits writer path=hdfs://localhost:45015/hbase/data/default/testSequentialEditLogSeqNum/220731e525589e0a967d40a4d4b18006/recovered.edits/0000000000000000003-wal.1733241431923.temp (wrote 15 edits, skipped 0 edits in 0 ms) at 1733241432230 (+12 ms)Rename recovered edits hdfs://localhost:45015/hbase/data/default/testSequentialEditLogSeqNum/220731e525589e0a967d40a4d4b18006/recovered.edits/0000000000000000003-wal.1733241431923.temp to hdfs://localhost:45015/hbase/data/default/testSequentialEditLogSeqNum/220731e525589e0a967d40a4d4b18006/recovered.edits/0000000000000000020 at 1733241432232 (+2 ms)Processed 17 edits across 1 Regions in 52 ms; skipped=2; WAL=hdfs://localhost:45015/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733241431913/wal.1733241431923, size=3.2 K, length=3256, corrupted=false, cancelled=false at 1733241432234 (+2 ms) 2024-12-03T15:57:12,251 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testSequentialEditLogSeqNum Thread=406 (was 401) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-365845159_22 at /127.0.0.1:50782 [Waiting for operation #18] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: sync.4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-365845159_22 at /127.0.0.1:40938 [Waiting for operation #9] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: sync.1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) - Thread LEAK? -, OpenFileDescriptor=957 (was 921) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=886 (was 886), ProcessCount=11 (was 11), AvailableMemoryMB=4978 (was 5014) 2024-12-03T15:57:12,266 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testRegionMadeOfBulkLoadedFilesOnly Thread=406, OpenFileDescriptor=957, MaxFileDescriptor=1048576, SystemLoadAverage=886, ProcessCount=11, AvailableMemoryMB=4976 2024-12-03T15:57:12,285 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-03T15:57:12,287 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-03T15:57:12,314 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-03T15:57:12,318 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-38573690, suffix=, logDir=hdfs://localhost:45015/hbase/WALs/hregion-38573690, archiveDir=hdfs://localhost:45015/hbase/oldWALs, maxLogs=32 2024-12-03T15:57:12,336 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-38573690/hregion-38573690.1733241432318, exclude list is [], retry=0 2024-12-03T15:57:12,339 DEBUG [AsyncFSWAL-17-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46585,DS-5c6771fd-75c7-4775-ad88-ae5bbe88de5c,DISK] 2024-12-03T15:57:12,339 DEBUG [AsyncFSWAL-17-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44827,DS-e007f1cc-fc1b-4368-9716-e11e4fe2f4eb,DISK] 2024-12-03T15:57:12,340 DEBUG [AsyncFSWAL-17-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40575,DS-309fe9b7-1997-44cb-b0c9-1ce3c04a71de,DISK] 2024-12-03T15:57:12,375 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-38573690/hregion-38573690.1733241432318 2024-12-03T15:57:12,375 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37355:37355),(127.0.0.1/127.0.0.1:46731:46731),(127.0.0.1/127.0.0.1:44513:44513)] 2024-12-03T15:57:12,375 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => caf5f26374f107efdaa78ad346c05d49, NAME => 'testRegionMadeOfBulkLoadedFilesOnly,,1733241432285.caf5f26374f107efdaa78ad346c05d49.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testRegionMadeOfBulkLoadedFilesOnly', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45015/hbase 2024-12-03T15:57:12,387 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46585 is added to blk_1073741879_1056 (size=70) 2024-12-03T15:57:12,388 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40575 is added to blk_1073741879_1056 (size=70) 2024-12-03T15:57:12,388 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44827 is added to blk_1073741879_1056 (size=70) 2024-12-03T15:57:12,389 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testRegionMadeOfBulkLoadedFilesOnly,,1733241432285.caf5f26374f107efdaa78ad346c05d49.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T15:57:12,391 INFO [StoreOpener-caf5f26374f107efdaa78ad346c05d49-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region caf5f26374f107efdaa78ad346c05d49 2024-12-03T15:57:12,392 INFO [StoreOpener-caf5f26374f107efdaa78ad346c05d49-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region caf5f26374f107efdaa78ad346c05d49 columnFamilyName a 2024-12-03T15:57:12,392 DEBUG [StoreOpener-caf5f26374f107efdaa78ad346c05d49-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T15:57:12,393 INFO [StoreOpener-caf5f26374f107efdaa78ad346c05d49-1 {}] regionserver.HStore(327): Store=caf5f26374f107efdaa78ad346c05d49/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T15:57:12,393 INFO [StoreOpener-caf5f26374f107efdaa78ad346c05d49-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region caf5f26374f107efdaa78ad346c05d49 2024-12-03T15:57:12,400 INFO [StoreOpener-caf5f26374f107efdaa78ad346c05d49-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region caf5f26374f107efdaa78ad346c05d49 columnFamilyName b 2024-12-03T15:57:12,400 DEBUG [StoreOpener-caf5f26374f107efdaa78ad346c05d49-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T15:57:12,400 INFO [StoreOpener-caf5f26374f107efdaa78ad346c05d49-1 {}] regionserver.HStore(327): Store=caf5f26374f107efdaa78ad346c05d49/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T15:57:12,401 INFO [StoreOpener-caf5f26374f107efdaa78ad346c05d49-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region caf5f26374f107efdaa78ad346c05d49 2024-12-03T15:57:12,402 INFO [StoreOpener-caf5f26374f107efdaa78ad346c05d49-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region caf5f26374f107efdaa78ad346c05d49 columnFamilyName c 2024-12-03T15:57:12,403 DEBUG [StoreOpener-caf5f26374f107efdaa78ad346c05d49-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T15:57:12,406 INFO [StoreOpener-caf5f26374f107efdaa78ad346c05d49-1 {}] regionserver.HStore(327): Store=caf5f26374f107efdaa78ad346c05d49/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T15:57:12,406 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for caf5f26374f107efdaa78ad346c05d49 2024-12-03T15:57:12,407 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45015/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/caf5f26374f107efdaa78ad346c05d49 2024-12-03T15:57:12,408 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45015/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/caf5f26374f107efdaa78ad346c05d49 2024-12-03T15:57:12,409 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for caf5f26374f107efdaa78ad346c05d49 2024-12-03T15:57:12,409 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for caf5f26374f107efdaa78ad346c05d49 2024-12-03T15:57:12,410 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRegionMadeOfBulkLoadedFilesOnly descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-03T15:57:12,413 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for caf5f26374f107efdaa78ad346c05d49 2024-12-03T15:57:12,426 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45015/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/caf5f26374f107efdaa78ad346c05d49/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T15:57:12,427 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened caf5f26374f107efdaa78ad346c05d49; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59539970, jitterRate=-0.11278530955314636}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-03T15:57:12,428 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for caf5f26374f107efdaa78ad346c05d49: Writing region info on filesystem at 1733241432389Initializing all the Stores at 1733241432390 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733241432390Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733241432391 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733241432391Cleaning up temporary data from old regions at 1733241432409 (+18 ms)Region opened successfully at 1733241432428 (+19 ms) 2024-12-03T15:57:12,428 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing caf5f26374f107efdaa78ad346c05d49, disabling compactions & flushes 2024-12-03T15:57:12,429 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testRegionMadeOfBulkLoadedFilesOnly,,1733241432285.caf5f26374f107efdaa78ad346c05d49. 2024-12-03T15:57:12,429 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testRegionMadeOfBulkLoadedFilesOnly,,1733241432285.caf5f26374f107efdaa78ad346c05d49. 2024-12-03T15:57:12,429 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testRegionMadeOfBulkLoadedFilesOnly,,1733241432285.caf5f26374f107efdaa78ad346c05d49. after waiting 0 ms 2024-12-03T15:57:12,429 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testRegionMadeOfBulkLoadedFilesOnly,,1733241432285.caf5f26374f107efdaa78ad346c05d49. 2024-12-03T15:57:12,434 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testRegionMadeOfBulkLoadedFilesOnly,,1733241432285.caf5f26374f107efdaa78ad346c05d49. 2024-12-03T15:57:12,434 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for caf5f26374f107efdaa78ad346c05d49: Waiting for close lock at 1733241432428Disabling compacts and flushes for region at 1733241432428Disabling writes for close at 1733241432429 (+1 ms)Writing region close event to WAL at 1733241432434 (+5 ms)Closed at 1733241432434 2024-12-03T15:57:12,447 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46585 is added to blk_1073741878_1055 (size=175) 2024-12-03T15:57:12,447 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44827 is added to blk_1073741878_1055 (size=175) 2024-12-03T15:57:12,449 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40575 is added to blk_1073741878_1055 (size=175) 2024-12-03T15:57:12,457 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-03T15:57:12,457 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-38573690:(num 1733241432318) 2024-12-03T15:57:12,458 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-03T15:57:12,460 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:45015/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733241432284, archiveDir=hdfs://localhost:45015/hbase/oldWALs, maxLogs=32 2024-12-03T15:57:12,481 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733241432284/wal.1733241432461, exclude list is [], retry=0 2024-12-03T15:57:12,485 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44827,DS-e007f1cc-fc1b-4368-9716-e11e4fe2f4eb,DISK] 2024-12-03T15:57:12,485 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40575,DS-309fe9b7-1997-44cb-b0c9-1ce3c04a71de,DISK] 2024-12-03T15:57:12,485 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46585,DS-5c6771fd-75c7-4775-ad88-ae5bbe88de5c,DISK] 2024-12-03T15:57:12,518 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733241432284/wal.1733241432461 2024-12-03T15:57:12,518 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46731:46731),(127.0.0.1/127.0.0.1:44513:44513),(127.0.0.1/127.0.0.1:37355:37355)] 2024-12-03T15:57:12,518 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => caf5f26374f107efdaa78ad346c05d49, NAME => 'testRegionMadeOfBulkLoadedFilesOnly,,1733241432285.caf5f26374f107efdaa78ad346c05d49.', STARTKEY => '', ENDKEY => ''} 2024-12-03T15:57:12,519 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testRegionMadeOfBulkLoadedFilesOnly,,1733241432285.caf5f26374f107efdaa78ad346c05d49.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T15:57:12,519 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for caf5f26374f107efdaa78ad346c05d49 2024-12-03T15:57:12,519 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for caf5f26374f107efdaa78ad346c05d49 2024-12-03T15:57:12,521 INFO [StoreOpener-caf5f26374f107efdaa78ad346c05d49-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region caf5f26374f107efdaa78ad346c05d49 2024-12-03T15:57:12,522 INFO [StoreOpener-caf5f26374f107efdaa78ad346c05d49-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region caf5f26374f107efdaa78ad346c05d49 columnFamilyName a 2024-12-03T15:57:12,522 DEBUG [StoreOpener-caf5f26374f107efdaa78ad346c05d49-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T15:57:12,523 INFO [StoreOpener-caf5f26374f107efdaa78ad346c05d49-1 {}] regionserver.HStore(327): Store=caf5f26374f107efdaa78ad346c05d49/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T15:57:12,523 INFO [StoreOpener-caf5f26374f107efdaa78ad346c05d49-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region caf5f26374f107efdaa78ad346c05d49 2024-12-03T15:57:12,524 INFO [StoreOpener-caf5f26374f107efdaa78ad346c05d49-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region caf5f26374f107efdaa78ad346c05d49 columnFamilyName b 2024-12-03T15:57:12,524 DEBUG [StoreOpener-caf5f26374f107efdaa78ad346c05d49-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T15:57:12,525 INFO [StoreOpener-caf5f26374f107efdaa78ad346c05d49-1 {}] regionserver.HStore(327): Store=caf5f26374f107efdaa78ad346c05d49/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T15:57:12,525 INFO [StoreOpener-caf5f26374f107efdaa78ad346c05d49-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region caf5f26374f107efdaa78ad346c05d49 2024-12-03T15:57:12,526 INFO [StoreOpener-caf5f26374f107efdaa78ad346c05d49-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region caf5f26374f107efdaa78ad346c05d49 columnFamilyName c 2024-12-03T15:57:12,526 DEBUG [StoreOpener-caf5f26374f107efdaa78ad346c05d49-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T15:57:12,526 INFO [StoreOpener-caf5f26374f107efdaa78ad346c05d49-1 {}] regionserver.HStore(327): Store=caf5f26374f107efdaa78ad346c05d49/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T15:57:12,527 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for caf5f26374f107efdaa78ad346c05d49 2024-12-03T15:57:12,527 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45015/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/caf5f26374f107efdaa78ad346c05d49 2024-12-03T15:57:12,529 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45015/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/caf5f26374f107efdaa78ad346c05d49 2024-12-03T15:57:12,534 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for caf5f26374f107efdaa78ad346c05d49 2024-12-03T15:57:12,534 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for caf5f26374f107efdaa78ad346c05d49 2024-12-03T15:57:12,535 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRegionMadeOfBulkLoadedFilesOnly descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-03T15:57:12,537 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for caf5f26374f107efdaa78ad346c05d49 2024-12-03T15:57:12,538 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened caf5f26374f107efdaa78ad346c05d49; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59725072, jitterRate=-0.11002707481384277}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-03T15:57:12,539 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for caf5f26374f107efdaa78ad346c05d49: Writing region info on filesystem at 1733241432519Initializing all the Stores at 1733241432520 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733241432520Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733241432521 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733241432521Cleaning up temporary data from old regions at 1733241432534 (+13 ms)Region opened successfully at 1733241432539 (+5 ms) 2024-12-03T15:57:12,548 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45015/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile is 28, key is \x0D/a:a/1733241432547/Put/seqid=0 2024-12-03T15:57:12,556 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44827 is added to blk_1073741881_1058 (size=4826) 2024-12-03T15:57:12,556 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46585 is added to blk_1073741881_1058 (size=4826) 2024-12-03T15:57:12,556 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40575 is added to blk_1073741881_1058 (size=4826) 2024-12-03T15:57:12,957 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:45015/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile for inclusion in caf5f26374f107efdaa78ad346c05d49/a 2024-12-03T15:57:12,967 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first= last=z 2024-12-03T15:57:12,967 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-12-03T15:57:12,968 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for caf5f26374f107efdaa78ad346c05d49: 2024-12-03T15:57:12,970 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45015/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile as hdfs://localhost:45015/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/caf5f26374f107efdaa78ad346c05d49/a/71a1fc5e2cb44bc5993e1cd4d9e7fdb8_SeqId_3_ 2024-12-03T15:57:12,971 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:45015/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile into caf5f26374f107efdaa78ad346c05d49/a as hdfs://localhost:45015/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/caf5f26374f107efdaa78ad346c05d49/a/71a1fc5e2cb44bc5993e1cd4d9e7fdb8_SeqId_3_ - updating store file list. 2024-12-03T15:57:12,976 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for 71a1fc5e2cb44bc5993e1cd4d9e7fdb8_SeqId_3_: NONE, but ROW specified in column family configuration 2024-12-03T15:57:12,977 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:45015/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/caf5f26374f107efdaa78ad346c05d49/a/71a1fc5e2cb44bc5993e1cd4d9e7fdb8_SeqId_3_ into caf5f26374f107efdaa78ad346c05d49/a 2024-12-03T15:57:12,977 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:45015/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile into caf5f26374f107efdaa78ad346c05d49/a (new location: hdfs://localhost:45015/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/caf5f26374f107efdaa78ad346c05d49/a/71a1fc5e2cb44bc5993e1cd4d9e7fdb8_SeqId_3_) 2024-12-03T15:57:13,025 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:45015/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733241432284/wal.1733241432461, size=0 (0bytes) 2024-12-03T15:57:13,025 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:45015/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733241432284/wal.1733241432461 might be still open, length is 0 2024-12-03T15:57:13,025 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:45015/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733241432284/wal.1733241432461 2024-12-03T15:57:13,025 WARN [IPC Server handler 0 on default port 45015 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733241432284/wal.1733241432461 has not been closed. Lease recovery is in progress. RecoveryId = 1059 for block blk_1073741880_1057 2024-12-03T15:57:13,026 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:45015/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733241432284/wal.1733241432461 after 0ms 2024-12-03T15:57:14,064 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-365845159_22 at /127.0.0.1:58392 [Receiving block BP-1122350750-172.17.0.3-1733241413817:blk_1073741880_1057] {}] datanode.DataXceiver(331): 127.0.0.1:44827:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:58392 dst: /127.0.0.1:44827 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:44827 remote=/127.0.0.1:58392]. Total timeout mills is 60000, 58919 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T15:57:14,065 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-365845159_22 at /127.0.0.1:41016 [Receiving block BP-1122350750-172.17.0.3-1733241413817:blk_1073741880_1057] {}] datanode.DataXceiver(331): 127.0.0.1:40575:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:41016 dst: /127.0.0.1:40575 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T15:57:14,065 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-365845159_22 at /127.0.0.1:56466 [Receiving block BP-1122350750-172.17.0.3-1733241413817:blk_1073741880_1057] {}] datanode.DataXceiver(331): 127.0.0.1:46585:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:56466 dst: /127.0.0.1:46585 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T15:57:14,068 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44827 is added to blk_1073741880_1059 (size=606) 2024-12-03T15:57:14,068 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40575 is added to blk_1073741880_1059 (size=606) 2024-12-03T15:57:14,068 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46585 is added to blk_1073741880_1059 (size=606) 2024-12-03T15:57:17,027 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:45015/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733241432284/wal.1733241432461 after 4001ms 2024-12-03T15:57:17,073 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-03T15:57:17,073 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:45015/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733241432284/wal.1733241432461 took 4049ms 2024-12-03T15:57:17,076 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:45015/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733241432284/wal.1733241432461; continuing. 2024-12-03T15:57:17,077 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:45015/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733241432284/wal.1733241432461 so closing down 2024-12-03T15:57:17,077 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-03T15:57:17,079 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000005-wal.1733241432461.temp 2024-12-03T15:57:17,129 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:45015/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/caf5f26374f107efdaa78ad346c05d49/recovered.edits/0000000000000000005-wal.1733241432461.temp 2024-12-03T15:57:17,129 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-03T15:57:17,139 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40575 is added to blk_1073741882_1060 (size=329) 2024-12-03T15:57:17,139 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46585 is added to blk_1073741882_1060 (size=329) 2024-12-03T15:57:17,143 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44827 is added to blk_1073741882_1060 (size=329) 2024-12-03T15:57:17,144 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:45015/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/caf5f26374f107efdaa78ad346c05d49/recovered.edits/0000000000000000005-wal.1733241432461.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-12-03T15:57:17,147 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:45015/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/caf5f26374f107efdaa78ad346c05d49/recovered.edits/0000000000000000005-wal.1733241432461.temp to hdfs://localhost:45015/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/caf5f26374f107efdaa78ad346c05d49/recovered.edits/0000000000000000005 2024-12-03T15:57:17,147 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 2 edits across 1 Regions in 73 ms; skipped=1; WAL=hdfs://localhost:45015/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733241432284/wal.1733241432461, size=0, length=0, corrupted=false, cancelled=false 2024-12-03T15:57:17,147 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:45015/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733241432284/wal.1733241432461, journal: Splitting hdfs://localhost:45015/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733241432284/wal.1733241432461, size=0 (0bytes) at 1733241433025Finishing writing output for hdfs://localhost:45015/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733241432284/wal.1733241432461 so closing down at 1733241437077 (+4052 ms)Creating recovered edits writer path=hdfs://localhost:45015/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/caf5f26374f107efdaa78ad346c05d49/recovered.edits/0000000000000000005-wal.1733241432461.temp at 1733241437129 (+52 ms)3 split writer threads finished at 1733241437129Closed recovered edits writer path=hdfs://localhost:45015/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/caf5f26374f107efdaa78ad346c05d49/recovered.edits/0000000000000000005-wal.1733241432461.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1733241437144 (+15 ms)Rename recovered edits hdfs://localhost:45015/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/caf5f26374f107efdaa78ad346c05d49/recovered.edits/0000000000000000005-wal.1733241432461.temp to hdfs://localhost:45015/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/caf5f26374f107efdaa78ad346c05d49/recovered.edits/0000000000000000005 at 1733241437147 (+3 ms)Processed 2 edits across 1 Regions in 73 ms; skipped=1; WAL=hdfs://localhost:45015/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733241432284/wal.1733241432461, size=0, length=0, corrupted=false, cancelled=false at 1733241437147 2024-12-03T15:57:17,151 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:45015/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733241432284/wal.1733241432461 to hdfs://localhost:45015/hbase/oldWALs/wal.1733241432461 2024-12-03T15:57:17,152 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:45015/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/caf5f26374f107efdaa78ad346c05d49/recovered.edits/0000000000000000005 2024-12-03T15:57:17,152 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-03T15:57:17,155 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:45015/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733241432284, archiveDir=hdfs://localhost:45015/hbase/oldWALs, maxLogs=32 2024-12-03T15:57:17,176 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733241432284/wal.1733241437155, exclude list is [], retry=0 2024-12-03T15:57:17,180 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40575,DS-309fe9b7-1997-44cb-b0c9-1ce3c04a71de,DISK] 2024-12-03T15:57:17,180 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46585,DS-5c6771fd-75c7-4775-ad88-ae5bbe88de5c,DISK] 2024-12-03T15:57:17,181 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44827,DS-e007f1cc-fc1b-4368-9716-e11e4fe2f4eb,DISK] 2024-12-03T15:57:17,224 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733241432284/wal.1733241437155 2024-12-03T15:57:17,225 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44513:44513),(127.0.0.1/127.0.0.1:37355:37355),(127.0.0.1/127.0.0.1:46731:46731)] 2024-12-03T15:57:17,225 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => caf5f26374f107efdaa78ad346c05d49, NAME => 'testRegionMadeOfBulkLoadedFilesOnly,,1733241432285.caf5f26374f107efdaa78ad346c05d49.', STARTKEY => '', ENDKEY => ''} 2024-12-03T15:57:17,225 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testRegionMadeOfBulkLoadedFilesOnly,,1733241432285.caf5f26374f107efdaa78ad346c05d49.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T15:57:17,225 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for caf5f26374f107efdaa78ad346c05d49 2024-12-03T15:57:17,225 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for caf5f26374f107efdaa78ad346c05d49 2024-12-03T15:57:17,227 INFO [StoreOpener-caf5f26374f107efdaa78ad346c05d49-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region caf5f26374f107efdaa78ad346c05d49 2024-12-03T15:57:17,229 INFO [StoreOpener-caf5f26374f107efdaa78ad346c05d49-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region caf5f26374f107efdaa78ad346c05d49 columnFamilyName a 2024-12-03T15:57:17,229 DEBUG [StoreOpener-caf5f26374f107efdaa78ad346c05d49-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T15:57:17,236 DEBUG [StoreFileOpener-caf5f26374f107efdaa78ad346c05d49-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for 71a1fc5e2cb44bc5993e1cd4d9e7fdb8_SeqId_3_: NONE, but ROW specified in column family configuration 2024-12-03T15:57:17,236 DEBUG [StoreOpener-caf5f26374f107efdaa78ad346c05d49-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:45015/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/caf5f26374f107efdaa78ad346c05d49/a/71a1fc5e2cb44bc5993e1cd4d9e7fdb8_SeqId_3_ 2024-12-03T15:57:17,236 INFO [StoreOpener-caf5f26374f107efdaa78ad346c05d49-1 {}] regionserver.HStore(327): Store=caf5f26374f107efdaa78ad346c05d49/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T15:57:17,236 INFO [StoreOpener-caf5f26374f107efdaa78ad346c05d49-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region caf5f26374f107efdaa78ad346c05d49 2024-12-03T15:57:17,238 INFO [StoreOpener-caf5f26374f107efdaa78ad346c05d49-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region caf5f26374f107efdaa78ad346c05d49 columnFamilyName b 2024-12-03T15:57:17,238 DEBUG [StoreOpener-caf5f26374f107efdaa78ad346c05d49-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T15:57:17,238 INFO [StoreOpener-caf5f26374f107efdaa78ad346c05d49-1 {}] regionserver.HStore(327): Store=caf5f26374f107efdaa78ad346c05d49/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T15:57:17,238 INFO [StoreOpener-caf5f26374f107efdaa78ad346c05d49-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region caf5f26374f107efdaa78ad346c05d49 2024-12-03T15:57:17,239 INFO [StoreOpener-caf5f26374f107efdaa78ad346c05d49-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region caf5f26374f107efdaa78ad346c05d49 columnFamilyName c 2024-12-03T15:57:17,239 DEBUG [StoreOpener-caf5f26374f107efdaa78ad346c05d49-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T15:57:17,240 INFO [StoreOpener-caf5f26374f107efdaa78ad346c05d49-1 {}] regionserver.HStore(327): Store=caf5f26374f107efdaa78ad346c05d49/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T15:57:17,240 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for caf5f26374f107efdaa78ad346c05d49 2024-12-03T15:57:17,241 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45015/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/caf5f26374f107efdaa78ad346c05d49 2024-12-03T15:57:17,243 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:45015/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/caf5f26374f107efdaa78ad346c05d49 2024-12-03T15:57:17,244 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:45015/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/caf5f26374f107efdaa78ad346c05d49/recovered.edits/0000000000000000005 2024-12-03T15:57:17,273 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-03T15:57:17,274 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 1, skipped 0, firstSequenceIdInLog=5, maxSequenceIdInLog=5, path=hdfs://localhost:45015/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/caf5f26374f107efdaa78ad346c05d49/recovered.edits/0000000000000000005 2024-12-03T15:57:17,275 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing caf5f26374f107efdaa78ad346c05d49 3/3 column families, dataSize=58 B heapSize=904 B 2024-12-03T15:57:17,291 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45015/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/caf5f26374f107efdaa78ad346c05d49/.tmp/a/15dbe8d21c6d4b5784694e3fa4f2dfef is 62, key is testRegionMadeOfBulkLoadedFilesOnly/a:a/1733241432982/Put/seqid=0 2024-12-03T15:57:17,298 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44827 is added to blk_1073741884_1062 (size=5149) 2024-12-03T15:57:17,298 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46585 is added to blk_1073741884_1062 (size=5149) 2024-12-03T15:57:17,298 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40575 is added to blk_1073741884_1062 (size=5149) 2024-12-03T15:57:17,299 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58 B at sequenceid=5 (bloomFilter=true), to=hdfs://localhost:45015/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/caf5f26374f107efdaa78ad346c05d49/.tmp/a/15dbe8d21c6d4b5784694e3fa4f2dfef 2024-12-03T15:57:17,306 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45015/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/caf5f26374f107efdaa78ad346c05d49/.tmp/a/15dbe8d21c6d4b5784694e3fa4f2dfef as hdfs://localhost:45015/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/caf5f26374f107efdaa78ad346c05d49/a/15dbe8d21c6d4b5784694e3fa4f2dfef 2024-12-03T15:57:17,314 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45015/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/caf5f26374f107efdaa78ad346c05d49/a/15dbe8d21c6d4b5784694e3fa4f2dfef, entries=1, sequenceid=5, filesize=5.0 K 2024-12-03T15:57:17,315 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~58 B/58, heapSize ~376 B/376, currentSize=0 B/0 for caf5f26374f107efdaa78ad346c05d49 in 41ms, sequenceid=5, compaction requested=false; wal=null 2024-12-03T15:57:17,316 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:45015/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/caf5f26374f107efdaa78ad346c05d49/recovered.edits/0000000000000000005 2024-12-03T15:57:17,317 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for caf5f26374f107efdaa78ad346c05d49 2024-12-03T15:57:17,317 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for caf5f26374f107efdaa78ad346c05d49 2024-12-03T15:57:17,318 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRegionMadeOfBulkLoadedFilesOnly descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-03T15:57:17,320 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for caf5f26374f107efdaa78ad346c05d49 2024-12-03T15:57:17,323 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45015/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/caf5f26374f107efdaa78ad346c05d49/recovered.edits/5.seqid, newMaxSeqId=5, maxSeqId=1 2024-12-03T15:57:17,324 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened caf5f26374f107efdaa78ad346c05d49; next sequenceid=6; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68437678, jitterRate=0.01980087161064148}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-03T15:57:17,324 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for caf5f26374f107efdaa78ad346c05d49: Writing region info on filesystem at 1733241437226Initializing all the Stores at 1733241437227 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733241437227Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733241437227Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733241437227Obtaining lock to block concurrent updates at 1733241437275 (+48 ms)Preparing flush snapshotting stores in caf5f26374f107efdaa78ad346c05d49 at 1733241437275Finished memstore snapshotting testRegionMadeOfBulkLoadedFilesOnly,,1733241432285.caf5f26374f107efdaa78ad346c05d49., syncing WAL and waiting on mvcc, flushsize=dataSize=58, getHeapSize=856, getOffHeapSize=0, getCellsCount=1 at 1733241437275Flushing stores of testRegionMadeOfBulkLoadedFilesOnly,,1733241432285.caf5f26374f107efdaa78ad346c05d49. at 1733241437275Flushing caf5f26374f107efdaa78ad346c05d49/a: creating writer at 1733241437275Flushing caf5f26374f107efdaa78ad346c05d49/a: appending metadata at 1733241437290 (+15 ms)Flushing caf5f26374f107efdaa78ad346c05d49/a: closing flushed file at 1733241437290Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@11debe8: reopening flushed file at 1733241437305 (+15 ms)Finished flush of dataSize ~58 B/58, heapSize ~376 B/376, currentSize=0 B/0 for caf5f26374f107efdaa78ad346c05d49 in 41ms, sequenceid=5, compaction requested=false; wal=null at 1733241437315 (+10 ms)Cleaning up temporary data from old regions at 1733241437317 (+2 ms)Region opened successfully at 1733241437324 (+7 ms) 2024-12-03T15:57:17,328 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing caf5f26374f107efdaa78ad346c05d49, disabling compactions & flushes 2024-12-03T15:57:17,329 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testRegionMadeOfBulkLoadedFilesOnly,,1733241432285.caf5f26374f107efdaa78ad346c05d49. 2024-12-03T15:57:17,329 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testRegionMadeOfBulkLoadedFilesOnly,,1733241432285.caf5f26374f107efdaa78ad346c05d49. 2024-12-03T15:57:17,329 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testRegionMadeOfBulkLoadedFilesOnly,,1733241432285.caf5f26374f107efdaa78ad346c05d49. after waiting 0 ms 2024-12-03T15:57:17,329 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testRegionMadeOfBulkLoadedFilesOnly,,1733241432285.caf5f26374f107efdaa78ad346c05d49. 2024-12-03T15:57:17,330 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testRegionMadeOfBulkLoadedFilesOnly,,1733241432285.caf5f26374f107efdaa78ad346c05d49. 2024-12-03T15:57:17,330 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for caf5f26374f107efdaa78ad346c05d49: Waiting for close lock at 1733241437328Disabling compacts and flushes for region at 1733241437328Disabling writes for close at 1733241437329 (+1 ms)Writing region close event to WAL at 1733241437329Closed at 1733241437329 2024-12-03T15:57:17,336 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40575 is added to blk_1073741883_1061 (size=175) 2024-12-03T15:57:17,336 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46585 is added to blk_1073741883_1061 (size=175) 2024-12-03T15:57:17,336 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44827 is added to blk_1073741883_1061 (size=175) 2024-12-03T15:57:17,340 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-03T15:57:17,340 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1733241437155) 2024-12-03T15:57:17,358 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testRegionMadeOfBulkLoadedFilesOnly Thread=415 (was 406) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb3fe2df-6726-509d-596d-17caab0e5978/cluster_ef20cb83-4e1e-0871-5945-558338f0ff60/data/data3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-17-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1616464420) connection to localhost/127.0.0.1:45015 from jenkinstestRegionMadeOfBulkLoadedFilesOnly java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: LeaseRenewer:jenkinstestRegionMadeOfBulkLoadedFilesOnly@localhost:45015 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-670846540_22 at /127.0.0.1:58428 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-670846540_22 at /127.0.0.1:56472 [Waiting for operation #3] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-670846540_22 at /127.0.0.1:41064 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-17-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb3fe2df-6726-509d-596d-17caab0e5978/cluster_ef20cb83-4e1e-0871-5945-558338f0ff60/data/data4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-17-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1023 (was 957) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=855 (was 886), ProcessCount=11 (was 11), AvailableMemoryMB=4932 (was 4976) 2024-12-03T15:57:17,372 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsAfterRegionMovedWithMultiCF Thread=415, OpenFileDescriptor=1023, MaxFileDescriptor=1048576, SystemLoadAverage=855, ProcessCount=11, AvailableMemoryMB=4929 2024-12-03T15:57:17,390 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-03T15:57:17,395 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-03T15:57:17,401 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 713339a81dd9,34663,1733241417250 2024-12-03T15:57:17,404 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@47bb982b 2024-12-03T15:57:17,405 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-03T15:57:17,407 INFO [HMaster-EventLoopGroup-2-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52336, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-03T15:57:17,411 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34663 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'testReplayEditsAfterRegionMovedWithMultiCF', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-03T15:57:17,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34663 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF 2024-12-03T15:57:17,421 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_PRE_OPERATION 2024-12-03T15:57:17,423 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34663 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "testReplayEditsAfterRegionMovedWithMultiCF" procId is: 4 2024-12-03T15:57:17,424 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T15:57:17,427 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-03T15:57:17,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34663 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-03T15:57:17,453 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46585 is added to blk_1073741885_1063 (size=694) 2024-12-03T15:57:17,454 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44827 is added to blk_1073741885_1063 (size=694) 2024-12-03T15:57:17,455 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40575 is added to blk_1073741885_1063 (size=694) 2024-12-03T15:57:17,460 INFO [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 03c5565060bb604d3f080f88f20309db, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1733241437409.03c5565060bb604d3f080f88f20309db.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsAfterRegionMovedWithMultiCF', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45015/user/jenkins/test-data/c81ce3c5-e5b8-1c2f-869e-120d734cf311 2024-12-03T15:57:17,479 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40575 is added to blk_1073741886_1064 (size=77) 2024-12-03T15:57:17,480 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46585 is added to blk_1073741886_1064 (size=77) 2024-12-03T15:57:17,481 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44827 is added to blk_1073741886_1064 (size=77) 2024-12-03T15:57:17,485 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1733241437409.03c5565060bb604d3f080f88f20309db.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T15:57:17,485 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1722): Closing 03c5565060bb604d3f080f88f20309db, disabling compactions & flushes 2024-12-03T15:57:17,485 INFO [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1733241437409.03c5565060bb604d3f080f88f20309db. 2024-12-03T15:57:17,485 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733241437409.03c5565060bb604d3f080f88f20309db. 2024-12-03T15:57:17,485 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733241437409.03c5565060bb604d3f080f88f20309db. after waiting 0 ms 2024-12-03T15:57:17,485 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1733241437409.03c5565060bb604d3f080f88f20309db. 2024-12-03T15:57:17,485 INFO [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733241437409.03c5565060bb604d3f080f88f20309db. 2024-12-03T15:57:17,485 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1676): Region close journal for 03c5565060bb604d3f080f88f20309db: Waiting for close lock at 1733241437485Disabling compacts and flushes for region at 1733241437485Disabling writes for close at 1733241437485Writing region close event to WAL at 1733241437485Closed at 1733241437485 2024-12-03T15:57:17,487 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_ADD_TO_META 2024-12-03T15:57:17,493 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testReplayEditsAfterRegionMovedWithMultiCF,,1733241437409.03c5565060bb604d3f080f88f20309db.","families":{"info":[{"qualifier":"regioninfo","vlen":76,"tag":[],"timestamp":"1733241437487"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733241437487"}]},"ts":"1733241437487"} 2024-12-03T15:57:17,499 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-03T15:57:17,502 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-03T15:57:17,506 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testReplayEditsAfterRegionMovedWithMultiCF","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733241437502"}]},"ts":"1733241437502"} 2024-12-03T15:57:17,512 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testReplayEditsAfterRegionMovedWithMultiCF, state=ENABLING in hbase:meta 2024-12-03T15:57:17,513 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {713339a81dd9=0} racks are {/default-rack=0} 2024-12-03T15:57:17,519 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-12-03T15:57:17,519 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-12-03T15:57:17,520 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-12-03T15:57:17,520 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-03T15:57:17,520 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-03T15:57:17,520 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-12-03T15:57:17,520 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-03T15:57:17,520 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-03T15:57:17,520 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-12-03T15:57:17,520 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-12-03T15:57:17,522 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=03c5565060bb604d3f080f88f20309db, ASSIGN}] 2024-12-03T15:57:17,524 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=03c5565060bb604d3f080f88f20309db, ASSIGN 2024-12-03T15:57:17,526 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=03c5565060bb604d3f080f88f20309db, ASSIGN; state=OFFLINE, location=713339a81dd9,36387,1733241418383; forceNewPlan=false, retain=false 2024-12-03T15:57:17,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34663 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-03T15:57:17,680 INFO [713339a81dd9:34663 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-12-03T15:57:17,681 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=03c5565060bb604d3f080f88f20309db, regionState=OPENING, regionLocation=713339a81dd9,36387,1733241418383 2024-12-03T15:57:17,685 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=03c5565060bb604d3f080f88f20309db, ASSIGN because future has completed 2024-12-03T15:57:17,686 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 03c5565060bb604d3f080f88f20309db, server=713339a81dd9,36387,1733241418383}] 2024-12-03T15:57:17,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34663 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-03T15:57:17,779 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRegionMadeOfBulkLoadedFilesOnly 2024-12-03T15:57:17,779 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testRegionMadeOfBulkLoadedFilesOnly Metrics about Tables on a single HBase RegionServer 2024-12-03T15:57:17,780 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testSequentialEditLogSeqNum 2024-12-03T15:57:17,780 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testSequentialEditLogSeqNum Metrics about Tables on a single HBase RegionServer 2024-12-03T15:57:17,841 DEBUG [RSProcedureDispatcher-pool-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-03T15:57:17,843 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:41657, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-03T15:57:17,848 INFO [RS_OPEN_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1733241437409.03c5565060bb604d3f080f88f20309db. 2024-12-03T15:57:17,849 DEBUG [RS_OPEN_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 03c5565060bb604d3f080f88f20309db, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1733241437409.03c5565060bb604d3f080f88f20309db.', STARTKEY => '', ENDKEY => ''} 2024-12-03T15:57:17,849 DEBUG [RS_OPEN_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF 03c5565060bb604d3f080f88f20309db 2024-12-03T15:57:17,849 DEBUG [RS_OPEN_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1733241437409.03c5565060bb604d3f080f88f20309db.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T15:57:17,849 DEBUG [RS_OPEN_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 03c5565060bb604d3f080f88f20309db 2024-12-03T15:57:17,849 DEBUG [RS_OPEN_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 03c5565060bb604d3f080f88f20309db 2024-12-03T15:57:17,851 INFO [StoreOpener-03c5565060bb604d3f080f88f20309db-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 03c5565060bb604d3f080f88f20309db 2024-12-03T15:57:17,852 INFO [StoreOpener-03c5565060bb604d3f080f88f20309db-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 03c5565060bb604d3f080f88f20309db columnFamilyName cf1 2024-12-03T15:57:17,853 DEBUG [StoreOpener-03c5565060bb604d3f080f88f20309db-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T15:57:17,853 INFO [StoreOpener-03c5565060bb604d3f080f88f20309db-1 {}] regionserver.HStore(327): Store=03c5565060bb604d3f080f88f20309db/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T15:57:17,853 INFO [StoreOpener-03c5565060bb604d3f080f88f20309db-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 03c5565060bb604d3f080f88f20309db 2024-12-03T15:57:17,854 INFO [StoreOpener-03c5565060bb604d3f080f88f20309db-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 03c5565060bb604d3f080f88f20309db columnFamilyName cf2 2024-12-03T15:57:17,855 DEBUG [StoreOpener-03c5565060bb604d3f080f88f20309db-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T15:57:17,855 INFO [StoreOpener-03c5565060bb604d3f080f88f20309db-1 {}] regionserver.HStore(327): Store=03c5565060bb604d3f080f88f20309db/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T15:57:17,855 DEBUG [RS_OPEN_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 03c5565060bb604d3f080f88f20309db 2024-12-03T15:57:17,856 DEBUG [RS_OPEN_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45015/user/jenkins/test-data/c81ce3c5-e5b8-1c2f-869e-120d734cf311/data/default/testReplayEditsAfterRegionMovedWithMultiCF/03c5565060bb604d3f080f88f20309db 2024-12-03T15:57:17,856 DEBUG [RS_OPEN_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45015/user/jenkins/test-data/c81ce3c5-e5b8-1c2f-869e-120d734cf311/data/default/testReplayEditsAfterRegionMovedWithMultiCF/03c5565060bb604d3f080f88f20309db 2024-12-03T15:57:17,857 DEBUG [RS_OPEN_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 03c5565060bb604d3f080f88f20309db 2024-12-03T15:57:17,857 DEBUG [RS_OPEN_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 03c5565060bb604d3f080f88f20309db 2024-12-03T15:57:17,858 DEBUG [RS_OPEN_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-12-03T15:57:17,859 DEBUG [RS_OPEN_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 03c5565060bb604d3f080f88f20309db 2024-12-03T15:57:17,862 DEBUG [RS_OPEN_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45015/user/jenkins/test-data/c81ce3c5-e5b8-1c2f-869e-120d734cf311/data/default/testReplayEditsAfterRegionMovedWithMultiCF/03c5565060bb604d3f080f88f20309db/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T15:57:17,863 INFO [RS_OPEN_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 03c5565060bb604d3f080f88f20309db; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71589678, jitterRate=0.06676933169364929}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-12-03T15:57:17,863 DEBUG [RS_OPEN_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 03c5565060bb604d3f080f88f20309db 2024-12-03T15:57:17,864 DEBUG [RS_OPEN_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 03c5565060bb604d3f080f88f20309db: Running coprocessor pre-open hook at 1733241437849Writing region info on filesystem at 1733241437849Initializing all the Stores at 1733241437850 (+1 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733241437850Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733241437851 (+1 ms)Cleaning up temporary data from old regions at 1733241437857 (+6 ms)Running coprocessor post-open hooks at 1733241437863 (+6 ms)Region opened successfully at 1733241437864 (+1 ms) 2024-12-03T15:57:17,866 INFO [RS_OPEN_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1733241437409.03c5565060bb604d3f080f88f20309db., pid=6, masterSystemTime=1733241437840 2024-12-03T15:57:17,869 DEBUG [RS_OPEN_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1733241437409.03c5565060bb604d3f080f88f20309db. 2024-12-03T15:57:17,869 INFO [RS_OPEN_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1733241437409.03c5565060bb604d3f080f88f20309db. 2024-12-03T15:57:17,870 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=03c5565060bb604d3f080f88f20309db, regionState=OPEN, openSeqNum=2, regionLocation=713339a81dd9,36387,1733241418383 2024-12-03T15:57:17,876 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 03c5565060bb604d3f080f88f20309db, server=713339a81dd9,36387,1733241418383 because future has completed 2024-12-03T15:57:17,880 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-12-03T15:57:17,881 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 03c5565060bb604d3f080f88f20309db, server=713339a81dd9,36387,1733241418383 in 191 msec 2024-12-03T15:57:17,884 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-12-03T15:57:17,884 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=03c5565060bb604d3f080f88f20309db, ASSIGN in 358 msec 2024-12-03T15:57:17,886 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-03T15:57:17,886 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testReplayEditsAfterRegionMovedWithMultiCF","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733241437886"}]},"ts":"1733241437886"} 2024-12-03T15:57:17,888 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testReplayEditsAfterRegionMovedWithMultiCF, state=ENABLED in hbase:meta 2024-12-03T15:57:17,890 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_POST_OPERATION 2024-12-03T15:57:17,893 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF in 477 msec 2024-12-03T15:57:18,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34663 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-03T15:57:18,062 INFO [RPCClient-NioEventLoopGroup-6-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testReplayEditsAfterRegionMovedWithMultiCF completed 2024-12-03T15:57:18,062 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testReplayEditsAfterRegionMovedWithMultiCF get assigned. Timeout = 60000ms 2024-12-03T15:57:18,064 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-03T15:57:18,072 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testReplayEditsAfterRegionMovedWithMultiCF assigned to meta. Checking AM states. 2024-12-03T15:57:18,072 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-03T15:57:18,073 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testReplayEditsAfterRegionMovedWithMultiCF assigned. 2024-12-03T15:57:18,093 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1733241437409.03c5565060bb604d3f080f88f20309db., hostname=713339a81dd9,36387,1733241418383, seqNum=2] 2024-12-03T15:57:18,094 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T15:57:18,097 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49830, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T15:57:18,115 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34663 {}] master.HMaster(2410): Client=jenkins//172.17.0.3 move hri=03c5565060bb604d3f080f88f20309db, source=713339a81dd9,36387,1733241418383, destination=713339a81dd9,42147,1733241418180, warming up region on 713339a81dd9,42147,1733241418180 2024-12-03T15:57:18,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34663 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-03T15:57:18,118 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34663 {}] master.HMaster(2414): Client=jenkins//172.17.0.3 move hri=03c5565060bb604d3f080f88f20309db, source=713339a81dd9,36387,1733241418383, destination=713339a81dd9,42147,1733241418180, running balancer 2024-12-03T15:57:18,120 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46637, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-03T15:57:18,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34663 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=03c5565060bb604d3f080f88f20309db, REOPEN/MOVE 2024-12-03T15:57:18,121 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=03c5565060bb604d3f080f88f20309db, REOPEN/MOVE 2024-12-03T15:57:18,123 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=03c5565060bb604d3f080f88f20309db, regionState=CLOSING, regionLocation=713339a81dd9,36387,1733241418383 2024-12-03T15:57:18,126 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=03c5565060bb604d3f080f88f20309db, REOPEN/MOVE because future has completed 2024-12-03T15:57:18,126 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42147 {}] regionserver.RSRpcServices(2066): Warmup testReplayEditsAfterRegionMovedWithMultiCF,,1733241437409.03c5565060bb604d3f080f88f20309db. 2024-12-03T15:57:18,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42147 {}] regionserver.HRegion(7855): Warmup {ENCODED => 03c5565060bb604d3f080f88f20309db, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1733241437409.03c5565060bb604d3f080f88f20309db.', STARTKEY => '', ENDKEY => ''} 2024-12-03T15:57:18,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42147 {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1733241437409.03c5565060bb604d3f080f88f20309db.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T15:57:18,128 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-03T15:57:18,128 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure 03c5565060bb604d3f080f88f20309db, server=713339a81dd9,36387,1733241418383}] 2024-12-03T15:57:18,142 INFO [StoreOpener-03c5565060bb604d3f080f88f20309db-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 03c5565060bb604d3f080f88f20309db 2024-12-03T15:57:18,144 INFO [StoreOpener-03c5565060bb604d3f080f88f20309db-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 03c5565060bb604d3f080f88f20309db columnFamilyName cf1 2024-12-03T15:57:18,145 DEBUG [StoreOpener-03c5565060bb604d3f080f88f20309db-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T15:57:18,145 INFO [StoreOpener-03c5565060bb604d3f080f88f20309db-1 {}] regionserver.HStore(327): Store=03c5565060bb604d3f080f88f20309db/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T15:57:18,146 INFO [StoreOpener-03c5565060bb604d3f080f88f20309db-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 03c5565060bb604d3f080f88f20309db 2024-12-03T15:57:18,147 INFO [StoreOpener-03c5565060bb604d3f080f88f20309db-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 03c5565060bb604d3f080f88f20309db columnFamilyName cf2 2024-12-03T15:57:18,147 DEBUG [StoreOpener-03c5565060bb604d3f080f88f20309db-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T15:57:18,149 INFO [StoreOpener-03c5565060bb604d3f080f88f20309db-1 {}] regionserver.HStore(327): Store=03c5565060bb604d3f080f88f20309db/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T15:57:18,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42147 {}] regionserver.HRegion(1722): Closing 03c5565060bb604d3f080f88f20309db, disabling compactions & flushes 2024-12-03T15:57:18,149 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42147 {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1733241437409.03c5565060bb604d3f080f88f20309db. 2024-12-03T15:57:18,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42147 {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733241437409.03c5565060bb604d3f080f88f20309db. 2024-12-03T15:57:18,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42147 {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733241437409.03c5565060bb604d3f080f88f20309db. after waiting 0 ms 2024-12-03T15:57:18,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42147 {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1733241437409.03c5565060bb604d3f080f88f20309db. 2024-12-03T15:57:18,150 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42147 {}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733241437409.03c5565060bb604d3f080f88f20309db. 2024-12-03T15:57:18,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42147 {}] regionserver.HRegion(1676): Region close journal for 03c5565060bb604d3f080f88f20309db: Waiting for close lock at 1733241438149Disabling compacts and flushes for region at 1733241438149Disabling writes for close at 1733241438149Writing region close event to WAL at 1733241438150 (+1 ms)Closed at 1733241438150 2024-12-03T15:57:18,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34663 {}] procedure.ProcedureSyncWait(219): waitFor pid=7 2024-12-03T15:57:18,206 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testReplayEditsAfterRegionMovedWithMultiCF' 2024-12-03T15:57:18,288 INFO [RS_CLOSE_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(122): Close 03c5565060bb604d3f080f88f20309db 2024-12-03T15:57:18,289 DEBUG [RS_CLOSE_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-03T15:57:18,289 DEBUG [RS_CLOSE_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1722): Closing 03c5565060bb604d3f080f88f20309db, disabling compactions & flushes 2024-12-03T15:57:18,289 INFO [RS_CLOSE_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1733241437409.03c5565060bb604d3f080f88f20309db. 2024-12-03T15:57:18,290 DEBUG [RS_CLOSE_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733241437409.03c5565060bb604d3f080f88f20309db. 2024-12-03T15:57:18,290 DEBUG [RS_CLOSE_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733241437409.03c5565060bb604d3f080f88f20309db. after waiting 0 ms 2024-12-03T15:57:18,290 DEBUG [RS_CLOSE_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1733241437409.03c5565060bb604d3f080f88f20309db. 2024-12-03T15:57:18,290 INFO [RS_CLOSE_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(2902): Flushing 03c5565060bb604d3f080f88f20309db 2/2 column families, dataSize=31 B heapSize=616 B 2024-12-03T15:57:18,316 DEBUG [RS_CLOSE_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45015/user/jenkins/test-data/c81ce3c5-e5b8-1c2f-869e-120d734cf311/data/default/testReplayEditsAfterRegionMovedWithMultiCF/03c5565060bb604d3f080f88f20309db/.tmp/cf1/669f3b842f844edcb5892c492e84b2db is 35, key is r1/cf1:q/1733241438097/Put/seqid=0 2024-12-03T15:57:18,329 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44827 is added to blk_1073741887_1065 (size=4783) 2024-12-03T15:57:18,330 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40575 is added to blk_1073741887_1065 (size=4783) 2024-12-03T15:57:18,331 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46585 is added to blk_1073741887_1065 (size=4783) 2024-12-03T15:57:18,331 INFO [RS_CLOSE_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31 B at sequenceid=5 (bloomFilter=false), to=hdfs://localhost:45015/user/jenkins/test-data/c81ce3c5-e5b8-1c2f-869e-120d734cf311/data/default/testReplayEditsAfterRegionMovedWithMultiCF/03c5565060bb604d3f080f88f20309db/.tmp/cf1/669f3b842f844edcb5892c492e84b2db 2024-12-03T15:57:18,340 DEBUG [RS_CLOSE_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45015/user/jenkins/test-data/c81ce3c5-e5b8-1c2f-869e-120d734cf311/data/default/testReplayEditsAfterRegionMovedWithMultiCF/03c5565060bb604d3f080f88f20309db/.tmp/cf1/669f3b842f844edcb5892c492e84b2db as hdfs://localhost:45015/user/jenkins/test-data/c81ce3c5-e5b8-1c2f-869e-120d734cf311/data/default/testReplayEditsAfterRegionMovedWithMultiCF/03c5565060bb604d3f080f88f20309db/cf1/669f3b842f844edcb5892c492e84b2db 2024-12-03T15:57:18,347 INFO [RS_CLOSE_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45015/user/jenkins/test-data/c81ce3c5-e5b8-1c2f-869e-120d734cf311/data/default/testReplayEditsAfterRegionMovedWithMultiCF/03c5565060bb604d3f080f88f20309db/cf1/669f3b842f844edcb5892c492e84b2db, entries=1, sequenceid=5, filesize=4.7 K 2024-12-03T15:57:18,348 INFO [RS_CLOSE_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~31 B/31, heapSize ~344 B/344, currentSize=0 B/0 for 03c5565060bb604d3f080f88f20309db in 58ms, sequenceid=5, compaction requested=false 2024-12-03T15:57:18,354 DEBUG [RS_CLOSE_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45015/user/jenkins/test-data/c81ce3c5-e5b8-1c2f-869e-120d734cf311/data/default/testReplayEditsAfterRegionMovedWithMultiCF/03c5565060bb604d3f080f88f20309db/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-12-03T15:57:18,357 INFO [RS_CLOSE_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733241437409.03c5565060bb604d3f080f88f20309db. 2024-12-03T15:57:18,357 DEBUG [RS_CLOSE_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1676): Region close journal for 03c5565060bb604d3f080f88f20309db: Waiting for close lock at 1733241438289Running coprocessor pre-close hooks at 1733241438289Disabling compacts and flushes for region at 1733241438289Disabling writes for close at 1733241438290 (+1 ms)Obtaining lock to block concurrent updates at 1733241438290Preparing flush snapshotting stores in 03c5565060bb604d3f080f88f20309db at 1733241438290Finished memstore snapshotting testReplayEditsAfterRegionMovedWithMultiCF,,1733241437409.03c5565060bb604d3f080f88f20309db., syncing WAL and waiting on mvcc, flushsize=dataSize=31, getHeapSize=584, getOffHeapSize=0, getCellsCount=1 at 1733241438290Flushing stores of testReplayEditsAfterRegionMovedWithMultiCF,,1733241437409.03c5565060bb604d3f080f88f20309db. at 1733241438291 (+1 ms)Flushing 03c5565060bb604d3f080f88f20309db/cf1: creating writer at 1733241438291Flushing 03c5565060bb604d3f080f88f20309db/cf1: appending metadata at 1733241438316 (+25 ms)Flushing 03c5565060bb604d3f080f88f20309db/cf1: closing flushed file at 1733241438316Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@51e2dc07: reopening flushed file at 1733241438338 (+22 ms)Finished flush of dataSize ~31 B/31, heapSize ~344 B/344, currentSize=0 B/0 for 03c5565060bb604d3f080f88f20309db in 58ms, sequenceid=5, compaction requested=false at 1733241438348 (+10 ms)Writing region close event to WAL at 1733241438349 (+1 ms)Running coprocessor post-close hooks at 1733241438355 (+6 ms)Closed at 1733241438357 (+2 ms) 2024-12-03T15:57:18,358 INFO [RS_CLOSE_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionServer(3302): Adding 03c5565060bb604d3f080f88f20309db move to 713339a81dd9,42147,1733241418180 record at close sequenceid=5 2024-12-03T15:57:18,361 INFO [RS_CLOSE_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(157): Closed 03c5565060bb604d3f080f88f20309db 2024-12-03T15:57:18,362 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=03c5565060bb604d3f080f88f20309db, regionState=CLOSED 2024-12-03T15:57:18,367 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure 03c5565060bb604d3f080f88f20309db, server=713339a81dd9,36387,1733241418383 because future has completed 2024-12-03T15:57:18,371 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-12-03T15:57:18,372 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; CloseRegionProcedure 03c5565060bb604d3f080f88f20309db, server=713339a81dd9,36387,1733241418383 in 240 msec 2024-12-03T15:57:18,372 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=03c5565060bb604d3f080f88f20309db, REOPEN/MOVE; state=CLOSED, location=713339a81dd9,42147,1733241418180; forceNewPlan=false, retain=false 2024-12-03T15:57:18,523 INFO [713339a81dd9:34663 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-12-03T15:57:18,524 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=03c5565060bb604d3f080f88f20309db, regionState=OPENING, regionLocation=713339a81dd9,42147,1733241418180 2024-12-03T15:57:18,527 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=03c5565060bb604d3f080f88f20309db, REOPEN/MOVE because future has completed 2024-12-03T15:57:18,528 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 03c5565060bb604d3f080f88f20309db, server=713339a81dd9,42147,1733241418180}] 2024-12-03T15:57:18,688 INFO [RS_OPEN_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1733241437409.03c5565060bb604d3f080f88f20309db. 2024-12-03T15:57:18,688 DEBUG [RS_OPEN_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => 03c5565060bb604d3f080f88f20309db, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1733241437409.03c5565060bb604d3f080f88f20309db.', STARTKEY => '', ENDKEY => ''} 2024-12-03T15:57:18,689 DEBUG [RS_OPEN_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF 03c5565060bb604d3f080f88f20309db 2024-12-03T15:57:18,689 DEBUG [RS_OPEN_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1733241437409.03c5565060bb604d3f080f88f20309db.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T15:57:18,689 DEBUG [RS_OPEN_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for 03c5565060bb604d3f080f88f20309db 2024-12-03T15:57:18,689 DEBUG [RS_OPEN_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for 03c5565060bb604d3f080f88f20309db 2024-12-03T15:57:18,694 INFO [StoreOpener-03c5565060bb604d3f080f88f20309db-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 03c5565060bb604d3f080f88f20309db 2024-12-03T15:57:18,696 INFO [StoreOpener-03c5565060bb604d3f080f88f20309db-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 03c5565060bb604d3f080f88f20309db columnFamilyName cf1 2024-12-03T15:57:18,696 DEBUG [StoreOpener-03c5565060bb604d3f080f88f20309db-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T15:57:18,708 DEBUG [StoreOpener-03c5565060bb604d3f080f88f20309db-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:45015/user/jenkins/test-data/c81ce3c5-e5b8-1c2f-869e-120d734cf311/data/default/testReplayEditsAfterRegionMovedWithMultiCF/03c5565060bb604d3f080f88f20309db/cf1/669f3b842f844edcb5892c492e84b2db 2024-12-03T15:57:18,708 INFO [StoreOpener-03c5565060bb604d3f080f88f20309db-1 {}] regionserver.HStore(327): Store=03c5565060bb604d3f080f88f20309db/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T15:57:18,708 INFO [StoreOpener-03c5565060bb604d3f080f88f20309db-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 03c5565060bb604d3f080f88f20309db 2024-12-03T15:57:18,710 INFO [StoreOpener-03c5565060bb604d3f080f88f20309db-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 03c5565060bb604d3f080f88f20309db columnFamilyName cf2 2024-12-03T15:57:18,710 DEBUG [StoreOpener-03c5565060bb604d3f080f88f20309db-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T15:57:18,711 INFO [StoreOpener-03c5565060bb604d3f080f88f20309db-1 {}] regionserver.HStore(327): Store=03c5565060bb604d3f080f88f20309db/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T15:57:18,711 DEBUG [RS_OPEN_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for 03c5565060bb604d3f080f88f20309db 2024-12-03T15:57:18,713 DEBUG [RS_OPEN_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45015/user/jenkins/test-data/c81ce3c5-e5b8-1c2f-869e-120d734cf311/data/default/testReplayEditsAfterRegionMovedWithMultiCF/03c5565060bb604d3f080f88f20309db 2024-12-03T15:57:18,714 DEBUG [RS_OPEN_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45015/user/jenkins/test-data/c81ce3c5-e5b8-1c2f-869e-120d734cf311/data/default/testReplayEditsAfterRegionMovedWithMultiCF/03c5565060bb604d3f080f88f20309db 2024-12-03T15:57:18,715 DEBUG [RS_OPEN_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for 03c5565060bb604d3f080f88f20309db 2024-12-03T15:57:18,715 DEBUG [RS_OPEN_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for 03c5565060bb604d3f080f88f20309db 2024-12-03T15:57:18,717 DEBUG [RS_OPEN_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-12-03T15:57:18,719 DEBUG [RS_OPEN_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for 03c5565060bb604d3f080f88f20309db 2024-12-03T15:57:18,720 INFO [RS_OPEN_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1114): Opened 03c5565060bb604d3f080f88f20309db; next sequenceid=9; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68280011, jitterRate=0.0174514502286911}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-12-03T15:57:18,721 DEBUG [RS_OPEN_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 03c5565060bb604d3f080f88f20309db 2024-12-03T15:57:18,722 DEBUG [RS_OPEN_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for 03c5565060bb604d3f080f88f20309db: Running coprocessor pre-open hook at 1733241438689Writing region info on filesystem at 1733241438689Initializing all the Stores at 1733241438691 (+2 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733241438691Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733241438694 (+3 ms)Cleaning up temporary data from old regions at 1733241438716 (+22 ms)Running coprocessor post-open hooks at 1733241438721 (+5 ms)Region opened successfully at 1733241438721 2024-12-03T15:57:18,726 INFO [RS_OPEN_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1733241437409.03c5565060bb604d3f080f88f20309db., pid=9, masterSystemTime=1733241438681 2024-12-03T15:57:18,731 DEBUG [RS_OPEN_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1733241437409.03c5565060bb604d3f080f88f20309db. 2024-12-03T15:57:18,731 INFO [RS_OPEN_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1733241437409.03c5565060bb604d3f080f88f20309db. 2024-12-03T15:57:18,732 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=03c5565060bb604d3f080f88f20309db, regionState=OPEN, openSeqNum=9, regionLocation=713339a81dd9,42147,1733241418180 2024-12-03T15:57:18,736 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 03c5565060bb604d3f080f88f20309db, server=713339a81dd9,42147,1733241418180 because future has completed 2024-12-03T15:57:18,737 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34663 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=713339a81dd9,42147,1733241418180, table=testReplayEditsAfterRegionMovedWithMultiCF, region=03c5565060bb604d3f080f88f20309db. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-12-03T15:57:18,745 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=7 2024-12-03T15:57:18,745 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=7, state=SUCCESS, hasLock=false; OpenRegionProcedure 03c5565060bb604d3f080f88f20309db, server=713339a81dd9,42147,1733241418180 in 212 msec 2024-12-03T15:57:18,749 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=03c5565060bb604d3f080f88f20309db, REOPEN/MOVE in 627 msec 2024-12-03T15:57:18,778 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-03T15:57:18,781 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49168, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-03T15:57:18,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36387 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 139 connection: 172.17.0.3:49830 deadline: 1733241498785, exception=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=713339a81dd9 port=42147 startCode=1733241418180. As of locationSeqNum=5. 2024-12-03T15:57:18,793 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733241437409.03c5565060bb604d3f080f88f20309db., hostname=713339a81dd9,36387,1733241418383, seqNum=2 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1733241437409.03c5565060bb604d3f080f88f20309db., hostname=713339a81dd9,36387,1733241418383, seqNum=2, error=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=713339a81dd9 port=42147 startCode=1733241418180. As of locationSeqNum=5. 2024-12-03T15:57:18,794 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733241437409.03c5565060bb604d3f080f88f20309db., hostname=713339a81dd9,36387,1733241418383, seqNum=2 is org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=713339a81dd9 port=42147 startCode=1733241418180. As of locationSeqNum=5. 2024-12-03T15:57:18,794 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(84): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733241437409.03c5565060bb604d3f080f88f20309db., hostname=713339a81dd9,36387,1733241418383, seqNum=2 with the new location region=testReplayEditsAfterRegionMovedWithMultiCF,,1733241437409.03c5565060bb604d3f080f88f20309db., hostname=713339a81dd9,42147,1733241418180, seqNum=5 constructed by org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=713339a81dd9 port=42147 startCode=1733241418180. As of locationSeqNum=5. 2024-12-03T15:57:18,901 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T15:57:18,904 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49172, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T15:57:18,917 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 03c5565060bb604d3f080f88f20309db 2/2 column families, dataSize=50 B heapSize=720 B 2024-12-03T15:57:18,943 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45015/user/jenkins/test-data/c81ce3c5-e5b8-1c2f-869e-120d734cf311/data/default/testReplayEditsAfterRegionMovedWithMultiCF/03c5565060bb604d3f080f88f20309db/.tmp/cf1/953842b9e72c4f078910bae2a5ccd306 is 29, key is r1/cf1:/1733241438906/DeleteFamily/seqid=0 2024-12-03T15:57:18,963 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44827 is added to blk_1073741888_1066 (size=4906) 2024-12-03T15:57:18,964 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40575 is added to blk_1073741888_1066 (size=4906) 2024-12-03T15:57:18,964 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46585 is added to blk_1073741888_1066 (size=4906) 2024-12-03T15:57:18,965 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=25 B at sequenceid=12 (bloomFilter=false), to=hdfs://localhost:45015/user/jenkins/test-data/c81ce3c5-e5b8-1c2f-869e-120d734cf311/data/default/testReplayEditsAfterRegionMovedWithMultiCF/03c5565060bb604d3f080f88f20309db/.tmp/cf1/953842b9e72c4f078910bae2a5ccd306 2024-12-03T15:57:18,978 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 953842b9e72c4f078910bae2a5ccd306 2024-12-03T15:57:19,001 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45015/user/jenkins/test-data/c81ce3c5-e5b8-1c2f-869e-120d734cf311/data/default/testReplayEditsAfterRegionMovedWithMultiCF/03c5565060bb604d3f080f88f20309db/.tmp/cf2/6c1873fed6fd4018abdb1c55ba514db9 is 29, key is r1/cf2:/1733241438906/DeleteFamily/seqid=0 2024-12-03T15:57:19,013 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46585 is added to blk_1073741889_1067 (size=4906) 2024-12-03T15:57:19,013 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40575 is added to blk_1073741889_1067 (size=4906) 2024-12-03T15:57:19,013 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44827 is added to blk_1073741889_1067 (size=4906) 2024-12-03T15:57:19,014 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=25 B at sequenceid=12 (bloomFilter=false), to=hdfs://localhost:45015/user/jenkins/test-data/c81ce3c5-e5b8-1c2f-869e-120d734cf311/data/default/testReplayEditsAfterRegionMovedWithMultiCF/03c5565060bb604d3f080f88f20309db/.tmp/cf2/6c1873fed6fd4018abdb1c55ba514db9 2024-12-03T15:57:19,024 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 6c1873fed6fd4018abdb1c55ba514db9 2024-12-03T15:57:19,025 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45015/user/jenkins/test-data/c81ce3c5-e5b8-1c2f-869e-120d734cf311/data/default/testReplayEditsAfterRegionMovedWithMultiCF/03c5565060bb604d3f080f88f20309db/.tmp/cf1/953842b9e72c4f078910bae2a5ccd306 as hdfs://localhost:45015/user/jenkins/test-data/c81ce3c5-e5b8-1c2f-869e-120d734cf311/data/default/testReplayEditsAfterRegionMovedWithMultiCF/03c5565060bb604d3f080f88f20309db/cf1/953842b9e72c4f078910bae2a5ccd306 2024-12-03T15:57:19,037 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 953842b9e72c4f078910bae2a5ccd306 2024-12-03T15:57:19,037 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45015/user/jenkins/test-data/c81ce3c5-e5b8-1c2f-869e-120d734cf311/data/default/testReplayEditsAfterRegionMovedWithMultiCF/03c5565060bb604d3f080f88f20309db/cf1/953842b9e72c4f078910bae2a5ccd306, entries=1, sequenceid=12, filesize=4.8 K 2024-12-03T15:57:19,039 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45015/user/jenkins/test-data/c81ce3c5-e5b8-1c2f-869e-120d734cf311/data/default/testReplayEditsAfterRegionMovedWithMultiCF/03c5565060bb604d3f080f88f20309db/.tmp/cf2/6c1873fed6fd4018abdb1c55ba514db9 as hdfs://localhost:45015/user/jenkins/test-data/c81ce3c5-e5b8-1c2f-869e-120d734cf311/data/default/testReplayEditsAfterRegionMovedWithMultiCF/03c5565060bb604d3f080f88f20309db/cf2/6c1873fed6fd4018abdb1c55ba514db9 2024-12-03T15:57:19,046 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 6c1873fed6fd4018abdb1c55ba514db9 2024-12-03T15:57:19,046 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45015/user/jenkins/test-data/c81ce3c5-e5b8-1c2f-869e-120d734cf311/data/default/testReplayEditsAfterRegionMovedWithMultiCF/03c5565060bb604d3f080f88f20309db/cf2/6c1873fed6fd4018abdb1c55ba514db9, entries=1, sequenceid=12, filesize=4.8 K 2024-12-03T15:57:19,048 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~50 B/50, heapSize ~688 B/688, currentSize=0 B/0 for 03c5565060bb604d3f080f88f20309db in 130ms, sequenceid=12, compaction requested=false 2024-12-03T15:57:19,048 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 03c5565060bb604d3f080f88f20309db: 2024-12-03T15:57:19,051 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-12-03T15:57:19,053 DEBUG [Time-limited test {}] regionserver.HStore(1541): 03c5565060bb604d3f080f88f20309db/cf1 is initiating major compaction (all files) 2024-12-03T15:57:19,053 INFO [Time-limited test {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-03T15:57:19,053 INFO [Time-limited test {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-03T15:57:19,053 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of 03c5565060bb604d3f080f88f20309db/cf1 in testReplayEditsAfterRegionMovedWithMultiCF,,1733241437409.03c5565060bb604d3f080f88f20309db. 2024-12-03T15:57:19,054 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45015/user/jenkins/test-data/c81ce3c5-e5b8-1c2f-869e-120d734cf311/data/default/testReplayEditsAfterRegionMovedWithMultiCF/03c5565060bb604d3f080f88f20309db/cf1/669f3b842f844edcb5892c492e84b2db, hdfs://localhost:45015/user/jenkins/test-data/c81ce3c5-e5b8-1c2f-869e-120d734cf311/data/default/testReplayEditsAfterRegionMovedWithMultiCF/03c5565060bb604d3f080f88f20309db/cf1/953842b9e72c4f078910bae2a5ccd306] into tmpdir=hdfs://localhost:45015/user/jenkins/test-data/c81ce3c5-e5b8-1c2f-869e-120d734cf311/data/default/testReplayEditsAfterRegionMovedWithMultiCF/03c5565060bb604d3f080f88f20309db/.tmp, totalSize=9.5 K 2024-12-03T15:57:19,058 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 669f3b842f844edcb5892c492e84b2db, keycount=1, bloomtype=NONE, size=4.7 K, encoding=NONE, compression=NONE, seqNum=5, earliestPutTs=1733241438097 2024-12-03T15:57:19,059 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 953842b9e72c4f078910bae2a5ccd306, keycount=1, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=9223372036854775807 2024-12-03T15:57:19,075 INFO [Time-limited test {}] throttle.PressureAwareThroughputController(145): 03c5565060bb604d3f080f88f20309db#cf1#compaction#16 average throughput is 0.00 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T15:57:19,086 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46585 is added to blk_1073741890_1068 (size=4626) 2024-12-03T15:57:19,086 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44827 is added to blk_1073741890_1068 (size=4626) 2024-12-03T15:57:19,086 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40575 is added to blk_1073741890_1068 (size=4626) 2024-12-03T15:57:19,495 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45015/user/jenkins/test-data/c81ce3c5-e5b8-1c2f-869e-120d734cf311/data/default/testReplayEditsAfterRegionMovedWithMultiCF/03c5565060bb604d3f080f88f20309db/.tmp/cf1/dcd12d26a4b94944866a71a0aeb0843b as hdfs://localhost:45015/user/jenkins/test-data/c81ce3c5-e5b8-1c2f-869e-120d734cf311/data/default/testReplayEditsAfterRegionMovedWithMultiCF/03c5565060bb604d3f080f88f20309db/cf1/dcd12d26a4b94944866a71a0aeb0843b 2024-12-03T15:57:19,513 INFO [Time-limited test {}] regionserver.HStore(1337): Completed major compaction of 2 (all) file(s) in 03c5565060bb604d3f080f88f20309db/cf1 of 03c5565060bb604d3f080f88f20309db into dcd12d26a4b94944866a71a0aeb0843b(size=4.5 K), total size for store is 4.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T15:57:19,513 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for 03c5565060bb604d3f080f88f20309db: 2024-12-03T15:57:19,514 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 1 store files, 0 compacting, 1 eligible, 16 blocking 2024-12-03T15:57:19,514 DEBUG [Time-limited test {}] regionserver.HStore(1541): 03c5565060bb604d3f080f88f20309db/cf2 is initiating major compaction (all files) 2024-12-03T15:57:19,514 INFO [Time-limited test {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-03T15:57:19,514 INFO [Time-limited test {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-03T15:57:19,514 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of 03c5565060bb604d3f080f88f20309db/cf2 in testReplayEditsAfterRegionMovedWithMultiCF,,1733241437409.03c5565060bb604d3f080f88f20309db. 2024-12-03T15:57:19,514 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45015/user/jenkins/test-data/c81ce3c5-e5b8-1c2f-869e-120d734cf311/data/default/testReplayEditsAfterRegionMovedWithMultiCF/03c5565060bb604d3f080f88f20309db/cf2/6c1873fed6fd4018abdb1c55ba514db9] into tmpdir=hdfs://localhost:45015/user/jenkins/test-data/c81ce3c5-e5b8-1c2f-869e-120d734cf311/data/default/testReplayEditsAfterRegionMovedWithMultiCF/03c5565060bb604d3f080f88f20309db/.tmp, totalSize=4.8 K 2024-12-03T15:57:19,515 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 6c1873fed6fd4018abdb1c55ba514db9, keycount=1, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=9223372036854775807 2024-12-03T15:57:19,525 INFO [Time-limited test {}] throttle.PressureAwareThroughputController(145): 03c5565060bb604d3f080f88f20309db#cf2#compaction#17 average throughput is NaN MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T15:57:19,538 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44827 is added to blk_1073741891_1069 (size=4592) 2024-12-03T15:57:19,540 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40575 is added to blk_1073741891_1069 (size=4592) 2024-12-03T15:57:19,541 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46585 is added to blk_1073741891_1069 (size=4592) 2024-12-03T15:57:19,550 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45015/user/jenkins/test-data/c81ce3c5-e5b8-1c2f-869e-120d734cf311/data/default/testReplayEditsAfterRegionMovedWithMultiCF/03c5565060bb604d3f080f88f20309db/.tmp/cf2/65f80788e0a94956ae1d7e1a3d0126a3 as hdfs://localhost:45015/user/jenkins/test-data/c81ce3c5-e5b8-1c2f-869e-120d734cf311/data/default/testReplayEditsAfterRegionMovedWithMultiCF/03c5565060bb604d3f080f88f20309db/cf2/65f80788e0a94956ae1d7e1a3d0126a3 2024-12-03T15:57:19,559 INFO [Time-limited test {}] regionserver.HStore(1337): Completed major compaction of 1 (all) file(s) in 03c5565060bb604d3f080f88f20309db/cf2 of 03c5565060bb604d3f080f88f20309db into 65f80788e0a94956ae1d7e1a3d0126a3(size=4.5 K), total size for store is 4.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T15:57:19,559 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for 03c5565060bb604d3f080f88f20309db: 2024-12-03T15:57:19,565 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34663 {}] master.HMaster(2410): Client=jenkins//172.17.0.3 move hri=03c5565060bb604d3f080f88f20309db, source=713339a81dd9,42147,1733241418180, destination=713339a81dd9,36387,1733241418383, warming up region on 713339a81dd9,36387,1733241418383 2024-12-03T15:57:19,565 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34663 {}] master.HMaster(2414): Client=jenkins//172.17.0.3 move hri=03c5565060bb604d3f080f88f20309db, source=713339a81dd9,42147,1733241418180, destination=713339a81dd9,36387,1733241418383, running balancer 2024-12-03T15:57:19,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34663 {}] procedure2.ProcedureExecutor(1139): Stored pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=03c5565060bb604d3f080f88f20309db, REOPEN/MOVE 2024-12-03T15:57:19,568 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=03c5565060bb604d3f080f88f20309db, REOPEN/MOVE 2024-12-03T15:57:19,570 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36387 {}] regionserver.RSRpcServices(2066): Warmup testReplayEditsAfterRegionMovedWithMultiCF,,1733241437409.03c5565060bb604d3f080f88f20309db. 2024-12-03T15:57:19,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36387 {}] regionserver.HRegion(7855): Warmup {ENCODED => 03c5565060bb604d3f080f88f20309db, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1733241437409.03c5565060bb604d3f080f88f20309db.', STARTKEY => '', ENDKEY => ''} 2024-12-03T15:57:19,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36387 {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1733241437409.03c5565060bb604d3f080f88f20309db.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T15:57:19,571 INFO [StoreOpener-03c5565060bb604d3f080f88f20309db-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 03c5565060bb604d3f080f88f20309db 2024-12-03T15:57:19,572 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=03c5565060bb604d3f080f88f20309db, regionState=CLOSING, regionLocation=713339a81dd9,42147,1733241418180 2024-12-03T15:57:19,573 INFO [StoreOpener-03c5565060bb604d3f080f88f20309db-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 03c5565060bb604d3f080f88f20309db columnFamilyName cf1 2024-12-03T15:57:19,573 DEBUG [StoreOpener-03c5565060bb604d3f080f88f20309db-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T15:57:19,576 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=03c5565060bb604d3f080f88f20309db, REOPEN/MOVE because future has completed 2024-12-03T15:57:19,577 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-03T15:57:19,577 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=11, ppid=10, state=RUNNABLE, hasLock=false; CloseRegionProcedure 03c5565060bb604d3f080f88f20309db, server=713339a81dd9,42147,1733241418180}] 2024-12-03T15:57:19,601 DEBUG [StoreOpener-03c5565060bb604d3f080f88f20309db-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:45015/user/jenkins/test-data/c81ce3c5-e5b8-1c2f-869e-120d734cf311/data/default/testReplayEditsAfterRegionMovedWithMultiCF/03c5565060bb604d3f080f88f20309db/cf1/669f3b842f844edcb5892c492e84b2db 2024-12-03T15:57:19,607 INFO [StoreFileOpener-03c5565060bb604d3f080f88f20309db-cf1-1 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 953842b9e72c4f078910bae2a5ccd306 2024-12-03T15:57:19,608 DEBUG [StoreOpener-03c5565060bb604d3f080f88f20309db-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:45015/user/jenkins/test-data/c81ce3c5-e5b8-1c2f-869e-120d734cf311/data/default/testReplayEditsAfterRegionMovedWithMultiCF/03c5565060bb604d3f080f88f20309db/cf1/953842b9e72c4f078910bae2a5ccd306 2024-12-03T15:57:19,614 DEBUG [StoreOpener-03c5565060bb604d3f080f88f20309db-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:45015/user/jenkins/test-data/c81ce3c5-e5b8-1c2f-869e-120d734cf311/data/default/testReplayEditsAfterRegionMovedWithMultiCF/03c5565060bb604d3f080f88f20309db/cf1/dcd12d26a4b94944866a71a0aeb0843b 2024-12-03T15:57:19,614 INFO [StoreOpener-03c5565060bb604d3f080f88f20309db-1 {}] regionserver.HStore(327): Store=03c5565060bb604d3f080f88f20309db/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T15:57:19,615 INFO [StoreOpener-03c5565060bb604d3f080f88f20309db-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 03c5565060bb604d3f080f88f20309db 2024-12-03T15:57:19,616 INFO [StoreOpener-03c5565060bb604d3f080f88f20309db-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 03c5565060bb604d3f080f88f20309db columnFamilyName cf2 2024-12-03T15:57:19,616 DEBUG [StoreOpener-03c5565060bb604d3f080f88f20309db-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T15:57:19,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34663 {}] procedure.ProcedureSyncWait(219): waitFor pid=10 2024-12-03T15:57:19,629 DEBUG [StoreOpener-03c5565060bb604d3f080f88f20309db-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:45015/user/jenkins/test-data/c81ce3c5-e5b8-1c2f-869e-120d734cf311/data/default/testReplayEditsAfterRegionMovedWithMultiCF/03c5565060bb604d3f080f88f20309db/cf2/65f80788e0a94956ae1d7e1a3d0126a3 2024-12-03T15:57:19,634 INFO [StoreFileOpener-03c5565060bb604d3f080f88f20309db-cf2-1 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 6c1873fed6fd4018abdb1c55ba514db9 2024-12-03T15:57:19,635 DEBUG [StoreOpener-03c5565060bb604d3f080f88f20309db-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:45015/user/jenkins/test-data/c81ce3c5-e5b8-1c2f-869e-120d734cf311/data/default/testReplayEditsAfterRegionMovedWithMultiCF/03c5565060bb604d3f080f88f20309db/cf2/6c1873fed6fd4018abdb1c55ba514db9 2024-12-03T15:57:19,635 INFO [StoreOpener-03c5565060bb604d3f080f88f20309db-1 {}] regionserver.HStore(327): Store=03c5565060bb604d3f080f88f20309db/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T15:57:19,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36387 {}] regionserver.HRegion(1722): Closing 03c5565060bb604d3f080f88f20309db, disabling compactions & flushes 2024-12-03T15:57:19,635 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36387 {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1733241437409.03c5565060bb604d3f080f88f20309db. 2024-12-03T15:57:19,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36387 {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733241437409.03c5565060bb604d3f080f88f20309db. 2024-12-03T15:57:19,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36387 {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733241437409.03c5565060bb604d3f080f88f20309db. after waiting 0 ms 2024-12-03T15:57:19,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36387 {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1733241437409.03c5565060bb604d3f080f88f20309db. 2024-12-03T15:57:19,636 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36387 {}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733241437409.03c5565060bb604d3f080f88f20309db. 2024-12-03T15:57:19,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36387 {}] regionserver.HRegion(1676): Region close journal for 03c5565060bb604d3f080f88f20309db: Waiting for close lock at 1733241439635Disabling compacts and flushes for region at 1733241439635Disabling writes for close at 1733241439635Writing region close event to WAL at 1733241439636 (+1 ms)Closed at 1733241439636 2024-12-03T15:57:19,736 INFO [RS_CLOSE_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] handler.UnassignRegionHandler(122): Close 03c5565060bb604d3f080f88f20309db 2024-12-03T15:57:19,736 DEBUG [RS_CLOSE_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-03T15:57:19,736 DEBUG [RS_CLOSE_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1722): Closing 03c5565060bb604d3f080f88f20309db, disabling compactions & flushes 2024-12-03T15:57:19,736 INFO [RS_CLOSE_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1733241437409.03c5565060bb604d3f080f88f20309db. 2024-12-03T15:57:19,736 DEBUG [RS_CLOSE_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733241437409.03c5565060bb604d3f080f88f20309db. 2024-12-03T15:57:19,737 DEBUG [RS_CLOSE_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733241437409.03c5565060bb604d3f080f88f20309db. after waiting 0 ms 2024-12-03T15:57:19,737 DEBUG [RS_CLOSE_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1733241437409.03c5565060bb604d3f080f88f20309db. 2024-12-03T15:57:19,737 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1733241437409.03c5565060bb604d3f080f88f20309db.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45015/user/jenkins/test-data/c81ce3c5-e5b8-1c2f-869e-120d734cf311/data/default/testReplayEditsAfterRegionMovedWithMultiCF/03c5565060bb604d3f080f88f20309db/cf1/669f3b842f844edcb5892c492e84b2db, hdfs://localhost:45015/user/jenkins/test-data/c81ce3c5-e5b8-1c2f-869e-120d734cf311/data/default/testReplayEditsAfterRegionMovedWithMultiCF/03c5565060bb604d3f080f88f20309db/cf1/953842b9e72c4f078910bae2a5ccd306] to archive 2024-12-03T15:57:19,745 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1733241437409.03c5565060bb604d3f080f88f20309db.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-03T15:57:19,753 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1733241437409.03c5565060bb604d3f080f88f20309db.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45015/user/jenkins/test-data/c81ce3c5-e5b8-1c2f-869e-120d734cf311/data/default/testReplayEditsAfterRegionMovedWithMultiCF/03c5565060bb604d3f080f88f20309db/cf1/669f3b842f844edcb5892c492e84b2db to hdfs://localhost:45015/user/jenkins/test-data/c81ce3c5-e5b8-1c2f-869e-120d734cf311/archive/data/default/testReplayEditsAfterRegionMovedWithMultiCF/03c5565060bb604d3f080f88f20309db/cf1/669f3b842f844edcb5892c492e84b2db 2024-12-03T15:57:19,755 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1733241437409.03c5565060bb604d3f080f88f20309db.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45015/user/jenkins/test-data/c81ce3c5-e5b8-1c2f-869e-120d734cf311/data/default/testReplayEditsAfterRegionMovedWithMultiCF/03c5565060bb604d3f080f88f20309db/cf1/953842b9e72c4f078910bae2a5ccd306 to hdfs://localhost:45015/user/jenkins/test-data/c81ce3c5-e5b8-1c2f-869e-120d734cf311/archive/data/default/testReplayEditsAfterRegionMovedWithMultiCF/03c5565060bb604d3f080f88f20309db/cf1/953842b9e72c4f078910bae2a5ccd306 2024-12-03T15:57:19,775 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1733241437409.03c5565060bb604d3f080f88f20309db.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45015/user/jenkins/test-data/c81ce3c5-e5b8-1c2f-869e-120d734cf311/data/default/testReplayEditsAfterRegionMovedWithMultiCF/03c5565060bb604d3f080f88f20309db/cf2/6c1873fed6fd4018abdb1c55ba514db9] to archive 2024-12-03T15:57:19,776 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1733241437409.03c5565060bb604d3f080f88f20309db.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-03T15:57:19,778 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1733241437409.03c5565060bb604d3f080f88f20309db.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45015/user/jenkins/test-data/c81ce3c5-e5b8-1c2f-869e-120d734cf311/data/default/testReplayEditsAfterRegionMovedWithMultiCF/03c5565060bb604d3f080f88f20309db/cf2/6c1873fed6fd4018abdb1c55ba514db9 to hdfs://localhost:45015/user/jenkins/test-data/c81ce3c5-e5b8-1c2f-869e-120d734cf311/archive/data/default/testReplayEditsAfterRegionMovedWithMultiCF/03c5565060bb604d3f080f88f20309db/cf2/6c1873fed6fd4018abdb1c55ba514db9 2024-12-03T15:57:19,789 DEBUG [RS_CLOSE_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45015/user/jenkins/test-data/c81ce3c5-e5b8-1c2f-869e-120d734cf311/data/default/testReplayEditsAfterRegionMovedWithMultiCF/03c5565060bb604d3f080f88f20309db/recovered.edits/17.seqid, newMaxSeqId=17, maxSeqId=8 2024-12-03T15:57:19,789 INFO [RS_CLOSE_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733241437409.03c5565060bb604d3f080f88f20309db. 2024-12-03T15:57:19,790 DEBUG [RS_CLOSE_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1676): Region close journal for 03c5565060bb604d3f080f88f20309db: Waiting for close lock at 1733241439736Running coprocessor pre-close hooks at 1733241439736Disabling compacts and flushes for region at 1733241439736Disabling writes for close at 1733241439737 (+1 ms)Writing region close event to WAL at 1733241439781 (+44 ms)Running coprocessor post-close hooks at 1733241439789 (+8 ms)Closed at 1733241439789 2024-12-03T15:57:19,790 INFO [RS_CLOSE_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegionServer(3302): Adding 03c5565060bb604d3f080f88f20309db move to 713339a81dd9,36387,1733241418383 record at close sequenceid=12 2024-12-03T15:57:19,793 INFO [RS_CLOSE_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] handler.UnassignRegionHandler(157): Closed 03c5565060bb604d3f080f88f20309db 2024-12-03T15:57:19,796 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=03c5565060bb604d3f080f88f20309db, regionState=CLOSED 2024-12-03T15:57:19,810 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=10, state=RUNNABLE, hasLock=false; CloseRegionProcedure 03c5565060bb604d3f080f88f20309db, server=713339a81dd9,42147,1733241418180 because future has completed 2024-12-03T15:57:19,816 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=11, resume processing ppid=10 2024-12-03T15:57:19,816 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=10, state=SUCCESS, hasLock=false; CloseRegionProcedure 03c5565060bb604d3f080f88f20309db, server=713339a81dd9,42147,1733241418180 in 235 msec 2024-12-03T15:57:19,818 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=03c5565060bb604d3f080f88f20309db, REOPEN/MOVE; state=CLOSED, location=713339a81dd9,36387,1733241418383; forceNewPlan=false, retain=false 2024-12-03T15:57:19,968 INFO [713339a81dd9:34663 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-12-03T15:57:19,969 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=03c5565060bb604d3f080f88f20309db, regionState=OPENING, regionLocation=713339a81dd9,36387,1733241418383 2024-12-03T15:57:19,972 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=03c5565060bb604d3f080f88f20309db, REOPEN/MOVE because future has completed 2024-12-03T15:57:19,973 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 03c5565060bb604d3f080f88f20309db, server=713339a81dd9,36387,1733241418383}] 2024-12-03T15:57:20,131 INFO [RS_OPEN_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1733241437409.03c5565060bb604d3f080f88f20309db. 2024-12-03T15:57:20,131 DEBUG [RS_OPEN_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7752): Opening region: {ENCODED => 03c5565060bb604d3f080f88f20309db, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1733241437409.03c5565060bb604d3f080f88f20309db.', STARTKEY => '', ENDKEY => ''} 2024-12-03T15:57:20,132 DEBUG [RS_OPEN_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF 03c5565060bb604d3f080f88f20309db 2024-12-03T15:57:20,132 DEBUG [RS_OPEN_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1733241437409.03c5565060bb604d3f080f88f20309db.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T15:57:20,132 DEBUG [RS_OPEN_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7794): checking encryption for 03c5565060bb604d3f080f88f20309db 2024-12-03T15:57:20,132 DEBUG [RS_OPEN_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7797): checking classloading for 03c5565060bb604d3f080f88f20309db 2024-12-03T15:57:20,134 INFO [StoreOpener-03c5565060bb604d3f080f88f20309db-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 03c5565060bb604d3f080f88f20309db 2024-12-03T15:57:20,135 INFO [StoreOpener-03c5565060bb604d3f080f88f20309db-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 03c5565060bb604d3f080f88f20309db columnFamilyName cf1 2024-12-03T15:57:20,135 DEBUG [StoreOpener-03c5565060bb604d3f080f88f20309db-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T15:57:20,148 DEBUG [StoreOpener-03c5565060bb604d3f080f88f20309db-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:45015/user/jenkins/test-data/c81ce3c5-e5b8-1c2f-869e-120d734cf311/data/default/testReplayEditsAfterRegionMovedWithMultiCF/03c5565060bb604d3f080f88f20309db/cf1/dcd12d26a4b94944866a71a0aeb0843b 2024-12-03T15:57:20,149 INFO [StoreOpener-03c5565060bb604d3f080f88f20309db-1 {}] regionserver.HStore(327): Store=03c5565060bb604d3f080f88f20309db/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T15:57:20,149 INFO [StoreOpener-03c5565060bb604d3f080f88f20309db-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 03c5565060bb604d3f080f88f20309db 2024-12-03T15:57:20,150 INFO [StoreOpener-03c5565060bb604d3f080f88f20309db-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 03c5565060bb604d3f080f88f20309db columnFamilyName cf2 2024-12-03T15:57:20,151 DEBUG [StoreOpener-03c5565060bb604d3f080f88f20309db-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T15:57:20,164 DEBUG [StoreOpener-03c5565060bb604d3f080f88f20309db-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:45015/user/jenkins/test-data/c81ce3c5-e5b8-1c2f-869e-120d734cf311/data/default/testReplayEditsAfterRegionMovedWithMultiCF/03c5565060bb604d3f080f88f20309db/cf2/65f80788e0a94956ae1d7e1a3d0126a3 2024-12-03T15:57:20,164 INFO [StoreOpener-03c5565060bb604d3f080f88f20309db-1 {}] regionserver.HStore(327): Store=03c5565060bb604d3f080f88f20309db/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T15:57:20,164 DEBUG [RS_OPEN_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1038): replaying wal for 03c5565060bb604d3f080f88f20309db 2024-12-03T15:57:20,166 DEBUG [RS_OPEN_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45015/user/jenkins/test-data/c81ce3c5-e5b8-1c2f-869e-120d734cf311/data/default/testReplayEditsAfterRegionMovedWithMultiCF/03c5565060bb604d3f080f88f20309db 2024-12-03T15:57:20,167 DEBUG [RS_OPEN_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45015/user/jenkins/test-data/c81ce3c5-e5b8-1c2f-869e-120d734cf311/data/default/testReplayEditsAfterRegionMovedWithMultiCF/03c5565060bb604d3f080f88f20309db 2024-12-03T15:57:20,169 DEBUG [RS_OPEN_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1048): stopping wal replay for 03c5565060bb604d3f080f88f20309db 2024-12-03T15:57:20,169 DEBUG [RS_OPEN_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1060): Cleaning up temporary data for 03c5565060bb604d3f080f88f20309db 2024-12-03T15:57:20,170 DEBUG [RS_OPEN_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-12-03T15:57:20,172 DEBUG [RS_OPEN_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1093): writing seq id for 03c5565060bb604d3f080f88f20309db 2024-12-03T15:57:20,173 INFO [RS_OPEN_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1114): Opened 03c5565060bb604d3f080f88f20309db; next sequenceid=18; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66348277, jitterRate=-0.011333629488945007}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-12-03T15:57:20,173 DEBUG [RS_OPEN_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 03c5565060bb604d3f080f88f20309db 2024-12-03T15:57:20,174 DEBUG [RS_OPEN_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1006): Region open journal for 03c5565060bb604d3f080f88f20309db: Running coprocessor pre-open hook at 1733241440132Writing region info on filesystem at 1733241440132Initializing all the Stores at 1733241440133 (+1 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733241440133Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733241440134 (+1 ms)Cleaning up temporary data from old regions at 1733241440169 (+35 ms)Running coprocessor post-open hooks at 1733241440173 (+4 ms)Region opened successfully at 1733241440174 (+1 ms) 2024-12-03T15:57:20,175 INFO [RS_OPEN_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1733241437409.03c5565060bb604d3f080f88f20309db., pid=12, masterSystemTime=1733241440126 2024-12-03T15:57:20,181 DEBUG [RS_OPEN_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1733241437409.03c5565060bb604d3f080f88f20309db. 2024-12-03T15:57:20,182 INFO [RS_OPEN_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1733241437409.03c5565060bb604d3f080f88f20309db. 2024-12-03T15:57:20,182 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=03c5565060bb604d3f080f88f20309db, regionState=OPEN, openSeqNum=18, regionLocation=713339a81dd9,36387,1733241418383 2024-12-03T15:57:20,188 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=12, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 03c5565060bb604d3f080f88f20309db, server=713339a81dd9,36387,1733241418383 because future has completed 2024-12-03T15:57:20,198 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=10 2024-12-03T15:57:20,198 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=10, state=SUCCESS, hasLock=false; OpenRegionProcedure 03c5565060bb604d3f080f88f20309db, server=713339a81dd9,36387,1733241418383 in 220 msec 2024-12-03T15:57:20,200 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=03c5565060bb604d3f080f88f20309db, REOPEN/MOVE in 633 msec 2024-12-03T15:57:20,221 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-03T15:57:20,223 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34116, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-03T15:57:20,225 ERROR [Time-limited test {}] regionserver.HRegionServer(2442): ***** ABORTING region server 713339a81dd9,36387,1733241418383: testing ***** 2024-12-03T15:57:20,225 ERROR [Time-limited test {}] regionserver.HRegionServer(2447): RegionServer abort: loaded coprocessors are: [org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint] 2024-12-03T15:57:20,228 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for java.lang:type=Memory 2024-12-03T15:57:20,229 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=IPC 2024-12-03T15:57:20,235 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Replication 2024-12-03T15:57:20,237 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Server 2024-12-03T15:57:20,251 INFO [Time-limited test {}] regionserver.HRegionServer(2451): Dump of metrics as JSON on abort: { "beans": [ { "name": "java.lang:type=Memory", "modelerType": "sun.management.MemoryImpl", "ObjectPendingFinalizationCount": 0, "HeapMemoryUsage": { "committed": 1048576000, "init": 1048576000, "max": 2306867200, "used": 494586520 }, "NonHeapMemoryUsage": { "committed": 173080576, "init": 7667712, "max": -1, "used": 170464160 }, "Verbose": false, "ObjectName": "java.lang:type=Memory" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=IPC", "modelerType": "RegionServer,sub=IPC", "tag.Context": "regionserver", "tag.Hostname": "713339a81dd9", "queueSize": 0, "numCallsInGeneralQueue": 0, "numCallsInReplicationQueue": 0, "numCallsInBulkLoadQueue": 0, "numCallsInPriorityQueue": 0, "numCallsInMetaPriorityQueue": 0, "numOpenConnections": 3, "numActiveHandler": 0, "numActiveGeneralHandler": 0, "numActivePriorityHandler": 0, "numActiveReplicationHandler": 0, "numGeneralCallsDropped": 0, "numLifoModeSwitches": 0, "numCallsInWriteQueue": 0, "numActiveBulkLoadHandler": 0, "numCallsInReadQueue": 0, "numCallsInScanQueue": 0, "numActiveWriteHandler": 0, "numActiveReadHandler": 0, "numActiveScanHandler": 0, "nettyDirectMemoryUsage": 67108864, "nettyTotalPendingOutboundBytes": 0, "nettyMaxPendingOutboundBytes": 0, "receivedBytes": 11067, "exceptions.RegionMovedException": 1, "authenticationSuccesses": 0, "authorizationFailures": 0, "exceptions.requestTooBig": 0, "UnwritableTime_num_ops": 0, "UnwritableTime_min": 0, "UnwritableTime_max": 0, "UnwritableTime_mean": 0, "UnwritableTime_25th_percentile": 0, "UnwritableTime_median": 0, "UnwritableTime_75th_percentile": 0, "UnwritableTime_90th_percentile": 0, "UnwritableTime_95th_percentile": 0, "UnwritableTime_98th_percentile": 0, "UnwritableTime_99th_percentile": 0, "UnwritableTime_99.9th_percentile": 0, "exceptions.OutOfOrderScannerNextException": 0, "exceptions.rpcThrottling": 0, "exceptions.otherExceptions": 0, "ProcessCallTime_num_ops": 39, "ProcessCallTime_min": 0, "ProcessCallTime_max": 70, "ProcessCallTime_mean": 6, "ProcessCallTime_25th_percentile": 2, "ProcessCallTime_median": 3, "ProcessCallTime_75th_percentile": 5, "ProcessCallTime_90th_percentile": 6, "ProcessCallTime_95th_percentile": 30, "ProcessCallTime_98th_percentile": 70, "ProcessCallTime_99th_percentile": 70, "ProcessCallTime_99.9th_percentile": 70, "ProcessCallTime_TimeRangeCount_0-1": 7, "ProcessCallTime_TimeRangeCount_1-3": 12, "ProcessCallTime_TimeRangeCount_3-10": 9, "ProcessCallTime_TimeRangeCount_10-30": 2, "ProcessCallTime_TimeRangeCount_30-100": 1, "exceptions.callQueueTooBig": 0, "QueueCallTime_num_ops": 39, "QueueCallTime_min": 0, "QueueCallTime_max": 1, "QueueCallTime_mean": 0, "QueueCallTime_25th_percentile": 0, "QueueCallTime_median": 0, "QueueCallTime_75th_percentile": 0, "QueueCallTime_90th_percentile": 0, "QueueCallTime_95th_percentile": 1, "QueueCallTime_98th_percentile": 1, "QueueCallTime_99th_percentile": 1, "QueueCallTime_99.9th_percentile": 1, "QueueCallTime_TimeRangeCount_0-1": 31, "authenticationFailures": 0, "exceptions.multiResponseTooLarge": 0, "exceptions.callDropped": 0, "TotalCallTime_num_ops": 39, "TotalCallTime_min": 0, "TotalCallTime_max": 71, "TotalCallTime_mean": 6, "TotalCallTime_25th_percentile": 2, "TotalCallTime_median": 3, "TotalCallTime_75th_percentile": 5, "TotalCallTime_90th_percentile": 6, "TotalCallTime_95th_percentile": 30, "TotalCallTime_98th_percentile": 71, "TotalCallTime_99th_percentile": 71, "TotalCallTime_99.9th_percentile": 71, "TotalCallTime_TimeRangeCount_0-1": 6, "TotalCallTime_TimeRangeCount_1-3": 13, "TotalCallTime_TimeRangeCount_3-10": 9, "TotalCallTime_TimeRangeCount_10-30": 2, "TotalCallTime_TimeRangeCount_30-100": 1, "exceptions.RegionTooBusyException": 0, "exceptions.FailedSanityCheckException": 0, "ResponseSize_num_ops": 39, "ResponseSize_min": 0, "ResponseSize_max": 1139, "ResponseSize_mean": 159, "ResponseSize_25th_percentile": 2, "ResponseSize_median": 2, "ResponseSize_75th_percentile": 74, "ResponseSize_90th_percentile": 453, "ResponseSize_95th_percentile": 796, "ResponseSize_98th_percentile": 1001, "ResponseSize_99th_percentile": 1070, "ResponseSize_99.9th_percentile": 1132, "ResponseSize_SizeRangeCount_0-10": 22, "ResponseSize_SizeRangeCount_10-100": 5, "ResponseSize_SizeRangeCount_100-1000": 4, "exceptions.UnknownScannerException": 0, "exceptions": 1, "maxOutboundBytesExceeded": 0, "authenticationFallbacks": 0, "exceptions.quotaExceeded": 0, "exceptions.callTimedOut": 0, "exceptions.NotServingRegionException": 0, "authorizationSuccesses": 0, "exceptions.ScannerResetException": 0, "RequestSize_num_ops": 39, "RequestSize_min": 31, "RequestSize_max": 390, "RequestSize_mean": 204, "RequestSize_25th_percentile": 119, "RequestSize_median": 179, "RequestSize_75th_percentile": 330, "RequestSize_90th_percentile": 330, "RequestSize_95th_percentile": 365, "RequestSize_98th_percentile": 380, "RequestSize_99th_percentile": 385, "RequestSize_99.9th_percentile": 389, "RequestSize_SizeRangeCount_0-10": 2, "RequestSize_SizeRangeCount_100-1000": 29, "sentBytes": 5752 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Replication", "modelerType": "RegionServer,sub=Replication", "tag.Context": "regionserver", "tag.Hostname": "713339a81dd9", "source.shippedHFiles": 0, "Source.ageOfLastShippedOp_num_ops": 0, "Source.ageOfLastShippedOp_min": 0, "Source.ageOfLastShippedOp_max": 0, "Source.ageOfLastShippedOp_mean": 0, "Source.ageOfLastShippedOp_25th_percentile": 0, "Source.ageOfLastShippedOp_median": 0, "Source.ageOfLastShippedOp_75th_percentile": 0, "Source.ageOfLastShippedOp_90th_percentile": 0, "Source.ageOfLastShippedOp_95th_percentile": 0, "Source.ageOfLastShippedOp_98th_percentile": 0, "Source.ageOfLastShippedOp_99th_percentile": 0, "Source.ageOfLastShippedOp_99.9th_percentile": 0, "source.uncleanlyClosedLogs": 0, "source.closedLogsWithUnknownFileLength": 0, "source.walReaderEditsBufferUsage": 0, "source.repeatedLogFileBytes": 0, "source.sizeOfHFileRefsQueue": 0, "source.logReadInBytes": 0, "source.completedRecoverQueues": 0, "source.sizeOfLogQueue": 0, "source.restartedLogReading": 0, "source.failedRecoverQueues": 0, "source.ignoredUncleanlyClosedLogContentsInBytes": 0, "Sink.ageOfLastAppliedOp_num_ops": 0, "Sink.ageOfLastAppliedOp_min": 0, "Sink.ageOfLastAppliedOp_max": 0, "Sink.ageOfLastAppliedOp_mean": 0, "Sink.ageOfLastAppliedOp_25th_percentile": 0, "Sink.ageOfLastAppliedOp_median": 0, "Sink.ageOfLastAppliedOp_75th_percentile": 0, "Sink.ageOfLastAppliedOp_90th_percentile": 0, "Sink.ageOfLastAppliedOp_95th_percentile": 0, "Sink.ageOfLastAppliedOp_98th_percentile": 0, "Sink.ageOfLastAppliedOp_99th_percentile": 0, "Sink.ageOfLastAppliedOp_99.9th_percentile": 0, "source.logEditsRead": 0, "source.numInitializing": 0, "source.shippedOps": 0, "sink.appliedHFiles": 0, "source.logEditsFiltered": 0, "source.shippedBytes": 0, "sink.appliedOps": 0, "source.completedLogs": 0, "source.failedBatches": 0, "sink.failedBatches": 0, "source.shippedBatches": 0, "sink.appliedBatches": 0 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Server", "modelerType": "RegionServer,sub=Server", "tag.zookeeperQuorum": "127.0.0.1:61024", "tag.serverName": "713339a81dd9,42147,1733241418180", "tag.clusterId": "a4fd42ae-14ff-4383-be19-b09963b80309", "tag.Context": "regionserver", "tag.Hostname": "713339a81dd9", "regionCount": 0, "storeCount": 0, "hlogFileCount": 4, "hlogFileSize": 0, "storeFileCount": 0, "maxStoreFileCount": 0, "memStoreSize": 0, "memStoreHeapSize": 0, "memStoreOffHeapSize": 0, "storeFileSize": 0, "storeFileSizeGrowthRate": 0.0, "maxStoreFileAge": 0, "minStoreFileAge": 0, "avgStoreFileAge": 0, "numReferenceFiles": 0, "regionServerStartTime": 1733241418180, "averageRegionSize": 0, "storeFileIndexSize": 0, "staticIndexSize": 0, "staticBloomSize": 0, "bloomFilterRequestsCount": 0, "bloomFilterNegativeResultsCount": 0, "bloomFilterEligibleRequestsCount": 0, "mutationsWithoutWALCount": 0, "mutationsWithoutWALSize": 0, "percentFilesLocal": 0.0, "percentFilesLocalSecondaryRegions": 0.0, "totalBytesRead": 320685, "localBytesRead": 320685, "shortCircuitBytesRead": 0, "zeroCopyBytesRead": 0, "splitQueueLength": 0, "compactionQueueLength": 0, "smallCompactionQueueLength": 0, "largeCompactionQueueLength": 0, "flushQueueLength": 0, "blockCacheFreeSize": 922070024, "blockCacheCount": 0, "blockCacheDataBlockCount": 0, "blockCacheSize": 676856, "blockCacheCountHitPercent": 25.0, "blockCacheExpressHitPercent": 0.0, "l1CacheSize": 676856, "l1CacheFreeSize": 922070024, "l1CacheCount": 0, "l1CacheEvictionCount": 0, "l1CacheHitCount": 1, "l1CacheMissCount": 3, "l1CacheHitRatio": 0.25, "l1CacheMissRatio": 0.75, "l2CacheSize": 0, "l2CacheFreeSize": 0, "l2CacheCount": 0, "l2CacheEvictionCount": 0, "l2CacheHitCount": 0, "l2CacheMissCount": 0, "l2CacheHitRatio": 0.0, "l2CacheMissRatio": 0.0, "mobFileCacheCount": 0, "mobFileCacheHitPercent": 0.0, "readRequestRatePerSecond": 0.0, "writeRequestRatePerSecond": 0.0, "ByteBuffAllocatorHeapAllocationBytes": 227435, "ByteBuffAllocatorPoolAllocationBytes": 0, "ByteBuffAllocatorHeapAllocationRatio": 1.0, "ByteBuffAllocatorTotalBufferCount": 186, "ByteBuffAllocatorUsedBufferCount": 0, "activeScanners": 0, "totalRequestCount": 3, "totalRowActionRequestCount": 0, "readRequestCount": 0, "cpRequestCount": 0, "filteredReadRequestCount": 0, "writeRequestCount": 0, "rpcGetRequestCount": 0, "rpcFullScanRequestCount": 1, "rpcScanRequestCount": 1, "rpcMultiRequestCount": 0, "rpcMutateRequestCount": 1, "checkMutateFailedCount": 0, "checkMutatePassedCount": 0, "blockCacheHitCount": 1, "blockCacheHitCountPrimary": 1, "blockCacheHitCachingCount": 0, "blockCacheMissCount": 3, "blockCacheMissCountPrimary": 3, "blockCacheMissCachingCount": 1, "blockCacheEvictionCount": 0, "blockCacheEvictionCountPrimary": 0, "blockCacheFailedInsertionCount": 0, "blockCacheDataMissCount": 3, "blockCacheLeafIndexMissCount": 0, "blockCacheBloomChunkMissCount": 0, "blockCacheMetaMissCount": 0, "blockCacheRootIndexMissCount": 0, "blockCacheIntermediateIndexMissCount": 0, "blockCacheFileInfoMissCount": 0, "blockCacheGeneralBloomMetaMissCount": 0, "blockCacheDeleteFamilyBloomMissCount": 0, "blockCacheTrailerMissCount": 0, "blockCacheDataHitCount": 1, "blockCacheLeafIndexHitCount": 0, "blockCacheBloomChunkHitCount": 0, "blockCacheMetaHitCount": 0, "blockCacheRootIndexHitCount": 0, "blockCacheIntermediateIndexHitCount": 0, "blockCacheFileInfoHitCount": 0, "blockCacheGeneralBloomMetaHitCount": 0, "blockCacheDeleteFamilyBloomHitCount": 0, "blockCacheTrailerHitCount": 0, "updatesBlockedTime": 0, "flushedCellsCount": 0, "compactedCellsCount": 0, "majorCompactedCellsCount": 0, "flushedCellsSize": 0, "compactedCellsSize": 0, "majorCompactedCellsSize": 0, "cellsCountCompactedFromMob": 0, "cellsCountCompactedToMob": 0, "cellsSizeCompactedFromMob": 0, "cellsSizeCompactedToMob": 0, "mobFlushCount": 0, "mobFlushedCellsCount": 0, "mobFlushedCellsSize": 0, "mobScanCellsCount": 0, "mobScanCellsSize": 0, "mobFileCacheAccessCount": 0, "mobFileCacheMissCount": 0, "mobFileCacheEvictedCount": 0, "hedgedReads": 0, "hedgedReadWins": 0, "hedgedReadOpsInCurThread": 0, "blockedRequestCount": 0, "CheckAndMutate_num_ops": 0, "CheckAndMutate_min": 0, "CheckAndMutate_max": 0, "CheckAndMutate_mean": 0, "CheckAndMutate_25th_percentile": 0, "CheckAndMutate_median": 0, "CheckAndMutate_75th_percentile": 0, "CheckAndMutate_90th_percentile": 0, "CheckAndMutate_95th_percentile": 0, "CheckAndMutate_98th_percentile": 0, "CheckAndMutate_99th_percentile": 0, "CheckAndMutate_99.9th_percentile": 0, "MajorCompactionTime_num_ops": 2, "MajorCompactionTime_min": 45, "MajorCompactionTime_max": 456, "MajorCompactionTime_mean": 250, "MajorCompactionTime_25th_percentile": 45, "MajorCompactionTime_median": 46, "MajorCompactionTime_75th_percentile": 356, "MajorCompactionTime_90th_percentile": 416, "MajorCompactionTime_95th_percentile": 436, "MajorCompactionTime_98th_percentile": 448, "MajorCompactionTime_99th_percentile": 452, "MajorCompactionTime_99.9th_percentile": 455, "MajorCompactionTime_TimeRangeCount_30-100": 1, "MajorCompactionTime_TimeRangeCount_100-300": 1, "ScanTime_num_ops": 1, "ScanTime_min": 0, "ScanTime_max": 0, "ScanTime_mean": 0, "ScanTime_25th_percentile": 0, "ScanTime_median": 0, "ScanTime_75th_percentile": 0, "ScanTime_90th_percentile": 0, "ScanTime_95th_percentile": 0, "ScanTime_98th_percentile": 0, "ScanTime_99th_percentile": 0, "ScanTime_99.9th_percentile": 0, "ScanTime_TimeRangeCount_0-1": 1, "CheckAndMutateBlockBytesScanned_num_ops": 0, "CheckAndMutateBlockBytesScanned_min": 0, "CheckAndMutateBlockBytesScanned_max": 0, "CheckAndMutateBlockBytesScanned_mean": 0, "CheckAndMutateBlockBytesScanned_25th_percentile": 0, "CheckAndMutateBlockBytesScanned_median": 0, "CheckAndMutateBlockBytesScanned_75th_percentile": 0, "CheckAndMutateBlockBytesScanned_90th_percentile": 0, "CheckAndMutateBlockBytesScanned_95th_percentile": 0, "CheckAndMutateBlockBytesScanned_98th_percentile": 0, "CheckAndMutateBlockBytesScanned_99th_percentile": 0, "CheckAndMutateBlockBytesScanned_99.9th_percentile": 0, "Put_num_ops": 0, "Put_min": 0, "Put_max": 0, "Put_mean": 0, "Put_25th_percentile": 0, "Put_median": 0, "Put_75th_percentile": 0, "Put_90th_percentile": 0, "Put_95th_percentile": 0, "Put_98th_percentile": 0, "Put_99th_percentile": 0, "Put_99.9th_percentile": 0, "splitRequestCount": 0, "AppendBlockBytesScanned_num_ops": 0, "AppendBlockBytesScanned_min": 0, "AppendBlockBytesScanned_max": 0, "AppendBlockBytesScanned_mean": 0, "AppendBlockBytesScanned_25th_percentile": 0, "AppendBlockBytesScanned_median": 0, "AppendBlockBytesScanned_75th_percentile": 0, "AppendBlockBytesScanned_90th_percentile": 0, "AppendBlockBytesScanned_95th_percentile": 0, "AppendBlockBytesScanned_98th_percentile": 0, "AppendBlockBytesScanned_99th_percentile": 0, "AppendBlockBytesScanned_99.9th_percentile": 0, "PutBatch_num_ops": 0, "PutBatch_min": 0, "PutBatch_max": 0, "PutBatch_mean": 0, "PutBatch_25th_percentile": 0, "PutBatch_median": 0, "PutBatch_75th_percentile": 0, "PutBatch_90th_percentile": 0, "PutBatch_95th_percentile": 0, "PutBatch_98th_percentile": 0, "PutBatch_99th_percentile": 0, "PutBatch_99.9th_percentile": 0, "IncrementBlockBytesScanned_num_ops": 0, "IncrementBlockBytesScanned_min": 0, "IncrementBlockBytesScanned_max": 0, "IncrementBlockBytesScanned_mean": 0, "IncrementBlockBytesScanned_25th_percentile": 0, "IncrementBlockBytesScanned_median": 0, "IncrementBlockBytesScanned_75th_percentile": 0, "IncrementBlockBytesScanned_90th_percentile": 0, "IncrementBlockBytesScanned_95th_percentile": 0, "IncrementBlockBytesScanned_98th_percentile": 0, "IncrementBlockBytesScanned_99th_percentile": 0, "IncrementBlockBytesScanned_99.9th_percentile": 0, "SplitTime_num_ops": 0, "SplitTime_min": 0, "SplitTime_max": 0, "SplitTime_mean": 0, "SplitTime_25th_percentile": 0, "SplitTime_median": 0, "SplitTime_75th_percentile": 0, "SplitTime_90th_percentile": 0, "SplitTime_95th_percentile": 0, "SplitTime_98th_percentile": 0, "SplitTime_99th_percentile": 0, "SplitTime_99.9th_percentile": 0, "GetBlockBytesScanned_num_ops": 0, "GetBlockBytesScanned_min": 0, "GetBlockBytesScanned_max": 0, "GetBlockBytesScanned_mean": 0, "GetBlockBytesScanned_25th_percentile": 0, "GetBlockBytesScanned_median": 0, "GetBlockBytesScanned_75th_percentile": 0, "GetBlockBytesScanned_90th_percentile": 0, "GetBlockBytesScanned_95th_percentile": 0, "GetBlockBytesScanned_98th_percentile": 0, "GetBlockBytesScanned_99th_percentile": 0, "GetBlockBytesScanned_99.9th_percentile": 0, "majorCompactedInputBytes": 14595, "slowAppendCount": 0, "flushedOutputBytes": 9812, "Replay_num_ops": 0, "Replay_min": 0, "Replay_max": 0, "Replay_mean": 0, "Replay_25th_percentile": 0, "Replay_median": 0, "Replay_75th_percentile": 0, "Replay_90th_percentile": 0, "Replay_95th_percentile": 0, "Replay_98th_percentile": 0, "Replay_99th_percentile": 0, "Replay_99.9th_percentile": 0, "MajorCompactionInputSize_num_ops": 2, "MajorCompactionInputSize_min": 4906, "MajorCompactionInputSize_max": 9689, "MajorCompactionInputSize_mean": 7297, "MajorCompactionInputSize_25th_percentile": 6101, "MajorCompactionInputSize_median": 7297, "MajorCompactionInputSize_75th_percentile": 8493, "MajorCompactionInputSize_90th_percentile": 9210, "MajorCompactionInputSize_95th_percentile": 9449, "MajorCompactionInputSize_98th_percentile": 9593, "MajorCompactionInputSize_99th_percentile": 9641, "MajorCompactionInputSize_99.9th_percentile": 9684, "MajorCompactionInputSize_SizeRangeCount_100-1000": 2, "pauseInfoThresholdExceeded": 0, "CheckAndDelete_num_ops": 0, "CheckAndDelete_min": 0, "CheckAndDelete_max": 0, "CheckAndDelete_mean": 0, "CheckAndDelete_25th_percentile": 0, "CheckAndDelete_median": 0, "CheckAndDelete_75th_percentile": 0, "CheckAndDelete_90th_percentile": 0, "CheckAndDelete_95th_percentile": 0, "CheckAndDelete_98th_percentile": 0, "CheckAndDelete_99th_percentile": 0, "CheckAndDelete_99.9th_percentile": 0, "CompactionInputSize_num_ops": 2, "CompactionInputSize_min": 4906, "CompactionInputSize_max": 9689, "CompactionInputSize_mean": 7297, "CompactionInputSize_25th_percentile": 6101, "CompactionInputSize_median": 7297, "CompactionInputSize_75th_percentile": 8493, "CompactionInputSize_90th_percentile": 9210, "CompactionInputSize_95th_percentile": 9449, "CompactionInputSize_98th_percentile": 9593, "CompactionInputSize_99th_percentile": 9641, "CompactionInputSize_99.9th_percentile": 9684, "CompactionInputSize_SizeRangeCount_100-1000": 2, "flushedMemstoreBytes": 50, "majorCompactedOutputBytes": 9218, "slowPutCount": 0, "compactedInputBytes": 14595, "FlushOutputSize_num_ops": 1, "FlushOutputSize_min": 9812, "FlushOutputSize_max": 9812, "FlushOutputSize_mean": 9812, "FlushOutputSize_25th_percentile": 9812, "FlushOutputSize_median": 9812, "FlushOutputSize_75th_percentile": 9812, "FlushOutputSize_90th_percentile": 9812, "FlushOutputSize_95th_percentile": 9812, "FlushOutputSize_98th_percentile": 9812, "FlushOutputSize_99th_percentile": 9812, "FlushOutputSize_99.9th_percentile": 9812, "FlushOutputSize_SizeRangeCount_100-1000": 1, "PauseTimeWithGc_num_ops": 0, "PauseTimeWithGc_min": 0, "PauseTimeWithGc_max": 0, "PauseTimeWithGc_mean": 0, "PauseTimeWithGc_25th_percentile": 0, "PauseTimeWithGc_median": 0, "PauseTimeWithGc_75th_percentile": 0, "PauseTimeWithGc_90th_percentile": 0, "PauseTimeWithGc_95th_percentile": 0, "PauseTimeWithGc_98th_percentile": 0, "PauseTimeWithGc_99th_percentile": 0, "PauseTimeWithGc_99.9th_percentile": 0, "compactedOutputBytes": 9218, "pauseWarnThresholdExceeded": 0, "ScanBlockBytesScanned_num_ops": 1, "ScanBlockBytesScanned_min": 32, "ScanBlockBytesScanned_max": 32, "ScanBlockBytesScanned_mean": 32, "ScanBlockBytesScanned_25th_percentile": 32, "ScanBlockBytesScanned_median": 32, "ScanBlockBytesScanned_75th_percentile": 32, "ScanBlockBytesScanned_90th_percentile": 32, "ScanBlockBytesScanned_95th_percentile": 32, "ScanBlockBytesScanned_98th_percentile": 32, "ScanBlockBytesScanned_99th_percentile": 32, "ScanBlockBytesScanned_99.9th_percentile": 32, "ScanBlockBytesScanned_SizeRangeCount_10-100": 1, "Increment_num_ops": 0, "Increment_min": 0, "Increment_max": 0, "Increment_mean": 0, "Increment_25th_percentile": 0, "Increment_median": 0, "Increment_75th_percentile": 0, "Increment_90th_percentile": 0, "Increment_95th_percentile": 0, "Increment_98th_percentile": 0, "Increment_99th_percentile": 0, "Increment_99.9th_percentile": 0, "Delete_num_ops": 1, "Delete_min": 5, "Delete_max": 5, "Delete_mean": 5, "Delete_25th_percentile": 5, "Delete_median": 5, "Delete_75th_percentile": 5, "Delete_90th_percentile": 5, "Delete_95th_percentile": 5, "Delete_98th_percentile": 5, "Delete_99th_percentile": 5, "Delete_99.9th_percentile": 5, "Delete_TimeRangeCount_3-10": 1, "DeleteBatch_num_ops": 0, "DeleteBatch_min": 0, "DeleteBatch_max": 0, "DeleteBatch_mean": 0, "DeleteBatch_25th_percentile": 0, "DeleteBatch_median": 0, "DeleteBatch_75th_percentile": 0, "DeleteBatch_90th_percentile": 0, "DeleteBatch_95th_percentile": 0, "DeleteBatch_98th_percentile": 0, "DeleteBatch_99th_percentile": 0, "DeleteBatch_99.9th_percentile": 0, "blockBytesScannedCount": 32, "FlushMemstoreSize_num_ops": 1, "FlushMemstoreSize_min": 50, "FlushMemstoreSize_max": 50, "FlushMemstoreSize_mean": 50, "FlushMemstoreSize_25th_percentile": 50, "FlushMemstoreSize_median": 50, "FlushMemstoreSize_75th_percentile": 50, "FlushMemstoreSize_90th_percentile": 50, "FlushMemstoreSize_95th_percentile": 50, "FlushMemstoreSize_98th_percentile": 50, "FlushMemstoreSize_99th_percentile": 50, "FlushMemstoreSize_99.9th_percentile": 50, "FlushMemstoreSize_SizeRangeCount_10-100": 1, "CompactionInputFileCount_num_ops": 2, "CompactionInputFileCount_min": 1, "CompactionInputFileCount_max": 2, "CompactionInputFileCount_mean": 1, "CompactionInputFileCount_25th_percentile": 1, "CompactionInputFileCount_median": 2, "CompactionInputFileCount_75th_percentile": 2, "CompactionInputFileCount_90th_percentile": 2, "CompactionInputFileCount_95th_percentile": 2, "CompactionInputFileCount_98th_percentile": 2, "CompactionInputFileCount_99th_percentile": 2, "CompactionInputFileCount_99.9th_percentile": 2, "CompactionTime_num_ops": 2, "CompactionTime_min": 45, "CompactionTime_max": 456, "CompactionTime_mean": 250, "CompactionTime_25th_percentile": 45, "CompactionTime_median": 46, "CompactionTime_75th_percentile": 356, "CompactionTime_90th_percentile": 416, "CompactionTime_95th_percentile": 436, "CompactionTime_98th_percentile": 448, "CompactionTime_99th_percentile": 452, "CompactionTime_99.9th_percentile": 455, "CompactionTime_TimeRangeCount_30-100": 1, "CompactionTime_TimeRangeCount_100-300": 1, "Get_num_ops": 0, "Get_min": 0, "Get_max": 0, "Get_mean": 0, "Get_25th_percentile": 0, "Get_median": 0, "Get_75th_percentile": 0, "Get_90th_percentile": 0, "Get_95th_percentile": 0, "Get_98th_percentile": 0, "Get_99th_percentile": 0, "Get_99.9th_percentile": 0, "MajorCompactionInputFileCount_num_ops": 2, "MajorCompactionInputFileCount_min": 1, "MajorCompactionInputFileCount_max": 2, "MajorCompactionInputFileCount_mean": 1, "MajorCompactionInputFileCount_25th_percentile": 1, "MajorCompactionInputFileCount_median": 2, "MajorCompactionInputFileCount_75th_percentile": 2, "MajorCompactionInputFileCount_90th_percentile": 2, "MajorCompactionInputFileCount_95th_percentile": 2, "MajorCompactionInputFileCount_98th_percentile": 2, "MajorCompactionInputFileCount_99th_percentile": 2, "MajorCompactionInputFileCount_99.9th_percentile": 2, "scannerLeaseExpiredCount": 0, "CheckAndPut_num_ops": 0, "CheckAndPut_min": 0, "CheckAndPut_max": 0, "CheckAndPut_mean": 0, "CheckAndPut_25th_percentile": 0, "CheckAndPut_median": 0, "CheckAndPut_75th_percentile": 0, "CheckAndPut_90th_percentile": 0, "CheckAndPut_95th_percentile": 0, "CheckAndPut_98th_percentile": 0, "CheckAndPut_99th_percentile": 0, "CheckAndPut_99.9th_percentile": 0, "MajorCompactionOutputSize_num_ops": 2, "MajorCompactionOutputSize_min": 4592, "MajorCompactionOutputSize_max": 4626, "MajorCompactionOutputSize_mean": 4609, "MajorCompactionOutputSize_25th_percentile": 4600, "MajorCompactionOutputSize_median": 4609, "MajorCompactionOutputSize_75th_percentile": 4617, "MajorCompactionOutputSize_90th_percentile": 4622, "MajorCompactionOutputSize_95th_percentile": 4624, "MajorCompactionOutputSize_98th_percentile": 4625, "MajorCompactionOutputSize_99th_percentile": 4625, "MajorCompactionOutputSize_99.9th_percentile": 4625, "MajorCompactionOutputSize_SizeRangeCount_100-1000": 2, "CompactionOutputFileCount_num_ops": 2, "CompactionOutputFileCount_min": 1, "CompactionOutputFileCount_max": 1, "CompactionOutputFileCount_mean": 1, "CompactionOutputFileCount_25th_percentile": 1, "CompactionOutputFileCount_median": 1, "CompactionOutputFileCount_75th_percentile": 1, "CompactionOutputFileCount_90th_percentile": 1, "CompactionOutputFileCount_95th_percentile": 1, "CompactionOutputFileCount_98th_percentile": 1, "CompactionOutputFileCount_99th_percentile": 1, "CompactionOutputFileCount_99.9th_percentile": 1, "slowDeleteCount": 0, "FlushTime_num_ops": 1, "FlushTime_min": 130, "FlushTime_max": 130, "FlushTime_mean": 130, "FlushTime_25th_percentile": 130, "FlushTime_median": 130, "FlushTime_75th_percentile": 130, "FlushTime_90th_percentile": 130, "FlushTime_95th_percentile": 130, "FlushTime_98th_percentile": 130, "FlushTime_99th_percentile": 130, "FlushTime_99.9th_percentile": 130, "FlushTime_TimeRangeCount_100-300": 1, "splitSuccessCount": 0, "MajorCompactionOutputFileCount_num_ops": 2, "MajorCompactionOutputFileCount_min": 1, "MajorCompactionOutputFileCount_max": 1, "MajorCompactionOutputFileCount_mean": 1, "MajorCompactionOutputFileCount_25th_percentile": 1, "MajorCompactionOutputFileCount_median": 1, "MajorCompactionOutputFileCount_75th_percentile": 1, "MajorCompactionOutputFileCount_90th_percentile": 1, "MajorCompactionOutputFileCount_95th_percentile": 1, "MajorCompactionOutputFileCount_98th_percentile": 1, "MajorCompactionOutputFileCount_99th_percentile": 1, "MajorCompactionOutputFileCount_99.9th_percentile": 1, "slowGetCount": 0, "ScanSize_num_ops": 1, "ScanSize_min": 0, "ScanSize_max": 0, "ScanSize_mean": 0, "ScanSize_25th_percentile": 0, "ScanSize_median": 0, "ScanSize_75th_percentile": 0, "ScanSize_90th_percentile": 0, "ScanSize_95th_percentile": 0, "ScanSize_98th_percentile": 0, "ScanSize_99th_percentile": 0, "ScanSize_99.9th_percentile": 0, "ScanSize_SizeRangeCount_0-10": 1, "CompactionOutputSize_num_ops": 2, "CompactionOutputSize_min": 4592, "CompactionOutputSize_max": 4626, "CompactionOutputSize_mean": 4609, "CompactionOutputSize_25th_percentile": 4600, "CompactionOutputSize_median": 4609, "CompactionOutputSize_75th_percentile": 4617, "CompactionOutputSize_90th_percentile": 4622, "CompactionOutputSize_95th_percentile": 4624, "CompactionOutputSize_98th_percentile": 4625, "CompactionOutputSize_99th_percentile": 4625, "CompactionOutputSize_99.9th_percentile": 4625, "CompactionOutputSize_SizeRangeCount_100-1000": 2, "PauseTimeWithoutGc_num_ops": 0, "PauseTimeWithoutGc_min": 0, "PauseTimeWithoutGc_max": 0, "PauseTimeWithoutGc_mean": 0, "PauseTimeWithoutGc_25th_percentile": 0, "PauseTimeWithoutGc_median": 0, "PauseTimeWithoutGc_75th_percentile": 0, "PauseTimeWithoutGc_90th_percentile": 0, "PauseTimeWithoutGc_95th_percentile": 0, "PauseTimeWithoutGc_98th_percentile": 0, "PauseTimeWithoutGc_99th_percentile": 0, "PauseTimeWithoutGc_99.9th_percentile": 0, "slowIncrementCount": 0, "Append_num_ops": 0, "Append_min": 0, "Append_max": 0, "Append_mean": 0, "Append_25th_percentile": 0, "Append_median": 0, "Append_75th_percentile": 0, "Append_90th_percentile": 0, "Append_95th_percentile": 0, "Append_98th_percentile": 0, "Append_99th_percentile": 0, "Append_99.9th_percentile": 0, "Bulkload_count": 0, "Bulkload_mean_rate": 0.0, "Bulkload_1min_rate": 0.0, "Bulkload_5min_rate": 0.0, "Bulkload_15min_rate": 0.0, "Bulkload_num_ops": 0, "Bulkload_min": 0, "Bulkload_max": 0, "Bulkload_mean": 0, "Bulkload_25th_percentile": 0, "Bulkload_median": 0, "Bulkload_75th_percentile": 0, "Bulkload_90th_percentile": 0, "Bulkload_95th_percentile": 0, "Bulkload_98th_percentile": 0, "Bulkload_99th_percentile": 0, "Bulkload_99.9th_percentile": 0 } ] } 2024-12-03T15:57:20,256 WARN [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34663 {}] master.MasterRpcServices(700): 713339a81dd9,36387,1733241418383 reported a fatal error: ***** ABORTING region server 713339a81dd9,36387,1733241418383: testing ***** 2024-12-03T15:57:20,261 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '713339a81dd9,36387,1733241418383' ***** 2024-12-03T15:57:20,262 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: testing 2024-12-03T15:57:20,262 INFO [RS:2;713339a81dd9:36387 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-03T15:57:20,262 INFO [RS:2;713339a81dd9:36387 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager abruptly. 2024-12-03T15:57:20,263 INFO [RS:2;713339a81dd9:36387 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager abruptly. 2024-12-03T15:57:20,263 INFO [RS:2;713339a81dd9:36387 {}] regionserver.HRegionServer(3091): Received CLOSE for 03c5565060bb604d3f080f88f20309db 2024-12-03T15:57:20,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42147 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Get size: 140 connection: 172.17.0.3:49172 deadline: 1733241500263, exception=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=713339a81dd9 port=36387 startCode=1733241418383. As of locationSeqNum=12. 2024-12-03T15:57:20,263 INFO [RS:2;713339a81dd9:36387 {}] regionserver.HRegionServer(956): aborting server 713339a81dd9,36387,1733241418383 2024-12-03T15:57:20,263 INFO [RS:2;713339a81dd9:36387 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-03T15:57:20,264 DEBUG [RS_CLOSE_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 03c5565060bb604d3f080f88f20309db, disabling compactions & flushes 2024-12-03T15:57:20,264 INFO [RS:2;713339a81dd9:36387 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:2;713339a81dd9:36387. 2024-12-03T15:57:20,264 INFO [RS_CLOSE_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1733241437409.03c5565060bb604d3f080f88f20309db. 2024-12-03T15:57:20,264 DEBUG [RS_CLOSE_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733241437409.03c5565060bb604d3f080f88f20309db. 2024-12-03T15:57:20,264 DEBUG [RS_CLOSE_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733241437409.03c5565060bb604d3f080f88f20309db. after waiting 0 ms 2024-12-03T15:57:20,264 DEBUG [RS:2;713339a81dd9:36387 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-03T15:57:20,264 DEBUG [RS_CLOSE_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1733241437409.03c5565060bb604d3f080f88f20309db. 2024-12-03T15:57:20,264 DEBUG [RPCClient-NioEventLoopGroup-6-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733241437409.03c5565060bb604d3f080f88f20309db., hostname=713339a81dd9,42147,1733241418180, seqNum=5 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1733241437409.03c5565060bb604d3f080f88f20309db., hostname=713339a81dd9,42147,1733241418180, seqNum=5, error=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=713339a81dd9 port=36387 startCode=1733241418383. As of locationSeqNum=12. 2024-12-03T15:57:20,264 DEBUG [RPCClient-NioEventLoopGroup-6-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733241437409.03c5565060bb604d3f080f88f20309db., hostname=713339a81dd9,42147,1733241418180, seqNum=5 is org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=713339a81dd9 port=36387 startCode=1733241418383. As of locationSeqNum=12. 2024-12-03T15:57:20,264 DEBUG [RPCClient-NioEventLoopGroup-6-7 {}] client.AsyncRegionLocatorHelper(84): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733241437409.03c5565060bb604d3f080f88f20309db., hostname=713339a81dd9,42147,1733241418180, seqNum=5 with the new location region=testReplayEditsAfterRegionMovedWithMultiCF,,1733241437409.03c5565060bb604d3f080f88f20309db., hostname=713339a81dd9,36387,1733241418383, seqNum=12 constructed by org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=713339a81dd9 port=36387 startCode=1733241418383. As of locationSeqNum=12. 2024-12-03T15:57:20,264 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-03T15:57:20,265 DEBUG [RS:2;713339a81dd9:36387 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T15:57:20,265 INFO [RS:2;713339a81dd9:36387 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-12-03T15:57:20,265 DEBUG [RS:2;713339a81dd9:36387 {}] regionserver.HRegionServer(1325): Online Regions={03c5565060bb604d3f080f88f20309db=testReplayEditsAfterRegionMovedWithMultiCF,,1733241437409.03c5565060bb604d3f080f88f20309db.} 2024-12-03T15:57:20,266 DEBUG [RS:2;713339a81dd9:36387 {}] regionserver.HRegionServer(1351): Waiting on 03c5565060bb604d3f080f88f20309db 2024-12-03T15:57:20,281 INFO [RS_CLOSE_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733241437409.03c5565060bb604d3f080f88f20309db. 2024-12-03T15:57:20,281 DEBUG [RS_CLOSE_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 03c5565060bb604d3f080f88f20309db: Waiting for close lock at 1733241440263Running coprocessor pre-close hooks at 1733241440264 (+1 ms)Disabling compacts and flushes for region at 1733241440264Disabling writes for close at 1733241440264Writing region close event to WAL at 1733241440280 (+16 ms)Running coprocessor post-close hooks at 1733241440280Closed at 1733241440281 (+1 ms) 2024-12-03T15:57:20,281 DEBUG [RS_CLOSE_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733241437409.03c5565060bb604d3f080f88f20309db. 2024-12-03T15:57:20,311 INFO [regionserver/713339a81dd9:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-03T15:57:20,372 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server 713339a81dd9,36387,1733241418383 aborting at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processRequest(ServerRpcConnection.java:564) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processOneRpc(ServerRpcConnection.java:364) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyServerRpcConnection.process(NettyServerRpcConnection.java:89) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:56) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:31) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T15:57:20,373 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733241437409.03c5565060bb604d3f080f88f20309db., hostname=713339a81dd9,36387,1733241418383, seqNum=12 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1733241437409.03c5565060bb604d3f080f88f20309db., hostname=713339a81dd9,36387,1733241418383, seqNum=12, error=org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server 713339a81dd9,36387,1733241418383 aborting 2024-12-03T15:57:20,374 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733241437409.03c5565060bb604d3f080f88f20309db., hostname=713339a81dd9,36387,1733241418383, seqNum=12 is org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server 713339a81dd9,36387,1733241418383 aborting 2024-12-03T15:57:20,374 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testReplayEditsAfterRegionMovedWithMultiCF,,1733241437409.03c5565060bb604d3f080f88f20309db., hostname=713339a81dd9,36387,1733241418383, seqNum=12 from cache 2024-12-03T15:57:20,466 INFO [RS:2;713339a81dd9:36387 {}] regionserver.HRegionServer(976): stopping server 713339a81dd9,36387,1733241418383; all regions closed. 2024-12-03T15:57:20,470 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44827 is added to blk_1073741833_1009 (size=2288) 2024-12-03T15:57:20,470 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40575 is added to blk_1073741833_1009 (size=2288) 2024-12-03T15:57:20,470 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46585 is added to blk_1073741833_1009 (size=2288) 2024-12-03T15:57:20,471 DEBUG [RS:2;713339a81dd9:36387 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T15:57:20,471 INFO [RS:2;713339a81dd9:36387 {}] regionserver.LeaseManager(133): Closed leases 2024-12-03T15:57:20,471 INFO [RS:2;713339a81dd9:36387 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-03T15:57:20,471 INFO [RS:2;713339a81dd9:36387 {}] hbase.ChoreService(370): Chore service for: regionserver/713339a81dd9:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-12-03T15:57:20,472 INFO [regionserver/713339a81dd9:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-03T15:57:20,472 INFO [RS:2;713339a81dd9:36387 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-03T15:57:20,472 INFO [RS:2;713339a81dd9:36387 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-03T15:57:20,472 INFO [RS:2;713339a81dd9:36387 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-03T15:57:20,472 INFO [RS:2;713339a81dd9:36387 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-03T15:57:20,472 INFO [RS:2;713339a81dd9:36387 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:36387 2024-12-03T15:57:20,481 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34663-0x1017926911e0000, quorum=127.0.0.1:61024, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-03T15:57:20,481 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36387-0x1017926911e0003, quorum=127.0.0.1:61024, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/713339a81dd9,36387,1733241418383 2024-12-03T15:57:20,481 INFO [RS:2;713339a81dd9:36387 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-03T15:57:20,483 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [713339a81dd9,36387,1733241418383] 2024-12-03T15:57:20,491 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/713339a81dd9,36387,1733241418383 already deleted, retry=false 2024-12-03T15:57:20,491 INFO [RegionServerTracker-0 {}] master.ServerManager(695): Processing expiration of 713339a81dd9,36387,1733241418383 on 713339a81dd9,34663,1733241417250 2024-12-03T15:57:20,502 DEBUG [RegionServerTracker-0 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure 713339a81dd9,36387,1733241418383, splitWal=true, meta=false 2024-12-03T15:57:20,505 INFO [RegionServerTracker-0 {}] assignment.AssignmentManager(1999): Scheduled ServerCrashProcedure pid=13 for 713339a81dd9,36387,1733241418383 (carryingMeta=false) 713339a81dd9,36387,1733241418383/CRASHED/regionCount=1/lock=java.util.concurrent.locks.ReentrantReadWriteLock@418e462e[Write locks = 1, Read locks = 0], oldState=ONLINE. 2024-12-03T15:57:20,506 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(169): Start pid=13, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure 713339a81dd9,36387,1733241418383, splitWal=true, meta=false 2024-12-03T15:57:20,509 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(207): 713339a81dd9,36387,1733241418383 had 1 regions 2024-12-03T15:57:20,510 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=13, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure 713339a81dd9,36387,1733241418383, splitWal=true, meta=false, isMeta: false 2024-12-03T15:57:20,513 DEBUG [PEWorker-1 {}] master.MasterWalManager(329): Renamed region directory: hdfs://localhost:45015/user/jenkins/test-data/c81ce3c5-e5b8-1c2f-869e-120d734cf311/WALs/713339a81dd9,36387,1733241418383-splitting 2024-12-03T15:57:20,514 INFO [PEWorker-1 {}] master.SplitWALManager(105): 713339a81dd9,36387,1733241418383 WAL count=1, meta=false 2024-12-03T15:57:20,517 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure 713339a81dd9%2C36387%2C1733241418383.1733241420133}] 2024-12-03T15:57:20,525 DEBUG [PEWorker-2 {}] master.SplitWALManager(158): Acquired split WAL worker=713339a81dd9,34819,1733241418314 2024-12-03T15:57:20,527 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=15, ppid=14, state=RUNNABLE, hasLock=false; SplitWALRemoteProcedure 713339a81dd9%2C36387%2C1733241418383.1733241420133, worker=713339a81dd9,34819,1733241418314}] 2024-12-03T15:57:20,584 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36387-0x1017926911e0003, quorum=127.0.0.1:61024, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-03T15:57:20,585 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36387-0x1017926911e0003, quorum=127.0.0.1:61024, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-03T15:57:20,585 INFO [RS:2;713339a81dd9:36387 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-03T15:57:20,585 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1733241437409.03c5565060bb604d3f080f88f20309db., hostname=713339a81dd9,36387,1733241418383, seqNum=18] 2024-12-03T15:57:20,585 INFO [RS:2;713339a81dd9:36387 {}] regionserver.HRegionServer(1031): Exiting; stopping=713339a81dd9,36387,1733241418383; zookeeper connection closed. 2024-12-03T15:57:20,586 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@456d4137 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@456d4137 2024-12-03T15:57:20,587 WARN [RPCClient-NioEventLoopGroup-6-5 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 713339a81dd9:36387 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: 713339a81dd9/172.17.0.3:36387 Caused by: java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hbase.thirdparty.io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:336) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:339) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T15:57:20,588 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733241437409.03c5565060bb604d3f080f88f20309db., hostname=713339a81dd9,36387,1733241418383, seqNum=18 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1733241437409.03c5565060bb604d3f080f88f20309db., hostname=713339a81dd9,36387,1733241418383, seqNum=18, error=java.net.ConnectException: Call to address=713339a81dd9:36387 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: 713339a81dd9/172.17.0.3:36387 2024-12-03T15:57:20,588 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733241437409.03c5565060bb604d3f080f88f20309db., hostname=713339a81dd9,36387,1733241418383, seqNum=18 is java.net.ConnectException: Connection refused 2024-12-03T15:57:20,589 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testReplayEditsAfterRegionMovedWithMultiCF,,1733241437409.03c5565060bb604d3f080f88f20309db., hostname=713339a81dd9,36387,1733241418383, seqNum=18 from cache 2024-12-03T15:57:20,589 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] ipc.FailedServers(52): Added failed server with address 713339a81dd9:36387 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: 713339a81dd9/172.17.0.3:36387 2024-12-03T15:57:20,691 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34819 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SplitWALCallable, pid=15 2024-12-03T15:57:20,718 INFO [RS_LOG_REPLAY_OPS-regionserver/713339a81dd9:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(299): Splitting hdfs://localhost:45015/user/jenkins/test-data/c81ce3c5-e5b8-1c2f-869e-120d734cf311/WALs/713339a81dd9,36387,1733241418383-splitting/713339a81dd9%2C36387%2C1733241418383.1733241420133, size=2.2 K (2288bytes) 2024-12-03T15:57:20,718 INFO [RS_LOG_REPLAY_OPS-regionserver/713339a81dd9:0-0 {event_type=RS_LOG_REPLAY, pid=15}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:45015/user/jenkins/test-data/c81ce3c5-e5b8-1c2f-869e-120d734cf311/WALs/713339a81dd9,36387,1733241418383-splitting/713339a81dd9%2C36387%2C1733241418383.1733241420133 2024-12-03T15:57:20,719 INFO [RS_LOG_REPLAY_OPS-regionserver/713339a81dd9:0-0 {event_type=RS_LOG_REPLAY, pid=15}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:45015/user/jenkins/test-data/c81ce3c5-e5b8-1c2f-869e-120d734cf311/WALs/713339a81dd9,36387,1733241418383-splitting/713339a81dd9%2C36387%2C1733241418383.1733241420133 after 1ms 2024-12-03T15:57:20,755 DEBUG [RS_LOG_REPLAY_OPS-regionserver/713339a81dd9:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-03T15:57:20,756 INFO [RS_LOG_REPLAY_OPS-regionserver/713339a81dd9:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(310): Open hdfs://localhost:45015/user/jenkins/test-data/c81ce3c5-e5b8-1c2f-869e-120d734cf311/WALs/713339a81dd9,36387,1733241418383-splitting/713339a81dd9%2C36387%2C1733241418383.1733241420133 took 37ms 2024-12-03T15:57:20,764 DEBUG [RS_LOG_REPLAY_OPS-regionserver/713339a81dd9:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(352): Last flushed sequenceid for 03c5565060bb604d3f080f88f20309db: last_flushed_sequence_id: 12 store_sequence_id { family_name: "cf1" sequence_id: 12 } store_sequence_id { family_name: "cf2" sequence_id: 12 } 2024-12-03T15:57:20,764 DEBUG [RS_LOG_REPLAY_OPS-regionserver/713339a81dd9:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:45015/user/jenkins/test-data/c81ce3c5-e5b8-1c2f-869e-120d734cf311/WALs/713339a81dd9,36387,1733241418383-splitting/713339a81dd9%2C36387%2C1733241418383.1733241420133 so closing down 2024-12-03T15:57:20,765 DEBUG [RS_LOG_REPLAY_OPS-regionserver/713339a81dd9:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-03T15:57:20,765 INFO [RS_LOG_REPLAY_OPS-regionserver/713339a81dd9:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.OutputSink(145): 3 split writer threads finished 2024-12-03T15:57:20,765 INFO [RS_LOG_REPLAY_OPS-regionserver/713339a81dd9:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(425): Processed 6 edits across 0 Regions in 9 ms; skipped=6; WAL=hdfs://localhost:45015/user/jenkins/test-data/c81ce3c5-e5b8-1c2f-869e-120d734cf311/WALs/713339a81dd9,36387,1733241418383-splitting/713339a81dd9%2C36387%2C1733241418383.1733241420133, size=2.2 K, length=2288, corrupted=false, cancelled=false 2024-12-03T15:57:20,765 DEBUG [RS_LOG_REPLAY_OPS-regionserver/713339a81dd9:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(428): Completed split of hdfs://localhost:45015/user/jenkins/test-data/c81ce3c5-e5b8-1c2f-869e-120d734cf311/WALs/713339a81dd9,36387,1733241418383-splitting/713339a81dd9%2C36387%2C1733241418383.1733241420133, journal: Splitting hdfs://localhost:45015/user/jenkins/test-data/c81ce3c5-e5b8-1c2f-869e-120d734cf311/WALs/713339a81dd9,36387,1733241418383-splitting/713339a81dd9%2C36387%2C1733241418383.1733241420133, size=2.2 K (2288bytes) at 1733241440718Finishing writing output for hdfs://localhost:45015/user/jenkins/test-data/c81ce3c5-e5b8-1c2f-869e-120d734cf311/WALs/713339a81dd9,36387,1733241418383-splitting/713339a81dd9%2C36387%2C1733241418383.1733241420133 so closing down at 1733241440764 (+46 ms)3 split writer threads finished at 1733241440765 (+1 ms)Processed 6 edits across 0 Regions in 9 ms; skipped=6; WAL=hdfs://localhost:45015/user/jenkins/test-data/c81ce3c5-e5b8-1c2f-869e-120d734cf311/WALs/713339a81dd9,36387,1733241418383-splitting/713339a81dd9%2C36387%2C1733241418383.1733241420133, size=2.2 K, length=2288, corrupted=false, cancelled=false at 1733241440765 2024-12-03T15:57:20,765 DEBUG [RS_LOG_REPLAY_OPS-regionserver/713339a81dd9:0-0 {event_type=RS_LOG_REPLAY, pid=15}] regionserver.SplitLogWorker(218): Done splitting WAL hdfs://localhost:45015/user/jenkins/test-data/c81ce3c5-e5b8-1c2f-869e-120d734cf311/WALs/713339a81dd9,36387,1733241418383-splitting/713339a81dd9%2C36387%2C1733241418383.1733241420133 2024-12-03T15:57:20,766 DEBUG [RS_LOG_REPLAY_OPS-regionserver/713339a81dd9:0-0 {event_type=RS_LOG_REPLAY, pid=15}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=15 2024-12-03T15:57:20,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34663 {}] master.HMaster(4169): Remote procedure done, pid=15 2024-12-03T15:57:20,777 INFO [PEWorker-3 {}] wal.WALSplitUtil(143): Moved hdfs://localhost:45015/user/jenkins/test-data/c81ce3c5-e5b8-1c2f-869e-120d734cf311/WALs/713339a81dd9,36387,1733241418383-splitting/713339a81dd9%2C36387%2C1733241418383.1733241420133 to hdfs://localhost:45015/user/jenkins/test-data/c81ce3c5-e5b8-1c2f-869e-120d734cf311/oldWALs 2024-12-03T15:57:20,780 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=15, resume processing ppid=14 2024-12-03T15:57:20,780 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, ppid=14, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 713339a81dd9%2C36387%2C1733241418383.1733241420133, worker=713339a81dd9,34819,1733241418314 in 250 msec 2024-12-03T15:57:20,782 DEBUG [PEWorker-5 {}] master.SplitWALManager(172): Release split WAL worker=713339a81dd9,34819,1733241418314 2024-12-03T15:57:20,786 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-12-03T15:57:20,786 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; SplitWALProcedure 713339a81dd9%2C36387%2C1733241418383.1733241420133, worker=713339a81dd9,34819,1733241418314 in 267 msec 2024-12-03T15:57:20,789 INFO [PEWorker-1 {}] master.SplitLogManager(171): hdfs://localhost:45015/user/jenkins/test-data/c81ce3c5-e5b8-1c2f-869e-120d734cf311/WALs/713339a81dd9,36387,1733241418383-splitting dir is empty, no logs to split. 2024-12-03T15:57:20,789 INFO [PEWorker-1 {}] master.SplitWALManager(105): 713339a81dd9,36387,1733241418383 WAL count=0, meta=false 2024-12-03T15:57:20,789 DEBUG [PEWorker-1 {}] procedure.ServerCrashProcedure(329): Check if 713339a81dd9,36387,1733241418383 WAL splitting is done? wals=0, meta=false 2024-12-03T15:57:20,793 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(321): Remove WAL directory for 713339a81dd9,36387,1733241418383 failed, ignore...File hdfs://localhost:45015/user/jenkins/test-data/c81ce3c5-e5b8-1c2f-869e-120d734cf311/WALs/713339a81dd9,36387,1733241418383-splitting does not exist. 2024-12-03T15:57:20,795 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=03c5565060bb604d3f080f88f20309db, ASSIGN}] 2024-12-03T15:57:20,797 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=03c5565060bb604d3f080f88f20309db, ASSIGN 2024-12-03T15:57:20,798 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=03c5565060bb604d3f080f88f20309db, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-12-03T15:57:20,905 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1733241437409.03c5565060bb604d3f080f88f20309db., hostname=713339a81dd9,36387,1733241418383, seqNum=18] 2024-12-03T15:57:20,906 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] ipc.AbstractRpcClient(357): Not trying to connect to 713339a81dd9:36387 this server is in the failed servers list 2024-12-03T15:57:20,906 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733241437409.03c5565060bb604d3f080f88f20309db., hostname=713339a81dd9,36387,1733241418383, seqNum=18 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1733241437409.03c5565060bb604d3f080f88f20309db., hostname=713339a81dd9,36387,1733241418383, seqNum=18, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=713339a81dd9:36387 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 713339a81dd9:36387 2024-12-03T15:57:20,907 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733241437409.03c5565060bb604d3f080f88f20309db., hostname=713339a81dd9,36387,1733241418383, seqNum=18 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 713339a81dd9:36387 2024-12-03T15:57:20,907 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testReplayEditsAfterRegionMovedWithMultiCF,,1733241437409.03c5565060bb604d3f080f88f20309db., hostname=713339a81dd9,36387,1733241418383, seqNum=18 from cache 2024-12-03T15:57:20,949 DEBUG [713339a81dd9:34663 {}] balancer.BalancerClusterState(204): Hosts are {713339a81dd9=0} racks are {/default-rack=0} 2024-12-03T15:57:20,949 DEBUG [713339a81dd9:34663 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-12-03T15:57:20,949 DEBUG [713339a81dd9:34663 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-12-03T15:57:20,949 DEBUG [713339a81dd9:34663 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-03T15:57:20,949 DEBUG [713339a81dd9:34663 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-03T15:57:20,949 INFO [713339a81dd9:34663 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-03T15:57:20,949 INFO [713339a81dd9:34663 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-03T15:57:20,949 DEBUG [713339a81dd9:34663 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-12-03T15:57:20,950 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=03c5565060bb604d3f080f88f20309db, regionState=OPENING, regionLocation=713339a81dd9,42147,1733241418180 2024-12-03T15:57:20,953 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=03c5565060bb604d3f080f88f20309db, ASSIGN because future has completed 2024-12-03T15:57:20,954 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=17, ppid=16, state=RUNNABLE, hasLock=false; OpenRegionProcedure 03c5565060bb604d3f080f88f20309db, server=713339a81dd9,42147,1733241418180}] 2024-12-03T15:57:21,112 INFO [RS_OPEN_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1733241437409.03c5565060bb604d3f080f88f20309db. 2024-12-03T15:57:21,112 DEBUG [RS_OPEN_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(7752): Opening region: {ENCODED => 03c5565060bb604d3f080f88f20309db, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1733241437409.03c5565060bb604d3f080f88f20309db.', STARTKEY => '', ENDKEY => ''} 2024-12-03T15:57:21,113 DEBUG [RS_OPEN_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF 03c5565060bb604d3f080f88f20309db 2024-12-03T15:57:21,113 DEBUG [RS_OPEN_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1733241437409.03c5565060bb604d3f080f88f20309db.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T15:57:21,113 DEBUG [RS_OPEN_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(7794): checking encryption for 03c5565060bb604d3f080f88f20309db 2024-12-03T15:57:21,113 DEBUG [RS_OPEN_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(7797): checking classloading for 03c5565060bb604d3f080f88f20309db 2024-12-03T15:57:21,115 INFO [StoreOpener-03c5565060bb604d3f080f88f20309db-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 03c5565060bb604d3f080f88f20309db 2024-12-03T15:57:21,116 INFO [StoreOpener-03c5565060bb604d3f080f88f20309db-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 03c5565060bb604d3f080f88f20309db columnFamilyName cf1 2024-12-03T15:57:21,116 DEBUG [StoreOpener-03c5565060bb604d3f080f88f20309db-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T15:57:21,124 DEBUG [StoreOpener-03c5565060bb604d3f080f88f20309db-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:45015/user/jenkins/test-data/c81ce3c5-e5b8-1c2f-869e-120d734cf311/data/default/testReplayEditsAfterRegionMovedWithMultiCF/03c5565060bb604d3f080f88f20309db/cf1/dcd12d26a4b94944866a71a0aeb0843b 2024-12-03T15:57:21,124 INFO [StoreOpener-03c5565060bb604d3f080f88f20309db-1 {}] regionserver.HStore(327): Store=03c5565060bb604d3f080f88f20309db/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T15:57:21,125 INFO [StoreOpener-03c5565060bb604d3f080f88f20309db-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 03c5565060bb604d3f080f88f20309db 2024-12-03T15:57:21,126 INFO [StoreOpener-03c5565060bb604d3f080f88f20309db-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 03c5565060bb604d3f080f88f20309db columnFamilyName cf2 2024-12-03T15:57:21,126 DEBUG [StoreOpener-03c5565060bb604d3f080f88f20309db-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T15:57:21,136 DEBUG [StoreOpener-03c5565060bb604d3f080f88f20309db-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:45015/user/jenkins/test-data/c81ce3c5-e5b8-1c2f-869e-120d734cf311/data/default/testReplayEditsAfterRegionMovedWithMultiCF/03c5565060bb604d3f080f88f20309db/cf2/65f80788e0a94956ae1d7e1a3d0126a3 2024-12-03T15:57:21,136 INFO [StoreOpener-03c5565060bb604d3f080f88f20309db-1 {}] regionserver.HStore(327): Store=03c5565060bb604d3f080f88f20309db/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T15:57:21,136 DEBUG [RS_OPEN_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1038): replaying wal for 03c5565060bb604d3f080f88f20309db 2024-12-03T15:57:21,137 DEBUG [RS_OPEN_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45015/user/jenkins/test-data/c81ce3c5-e5b8-1c2f-869e-120d734cf311/data/default/testReplayEditsAfterRegionMovedWithMultiCF/03c5565060bb604d3f080f88f20309db 2024-12-03T15:57:21,139 DEBUG [RS_OPEN_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45015/user/jenkins/test-data/c81ce3c5-e5b8-1c2f-869e-120d734cf311/data/default/testReplayEditsAfterRegionMovedWithMultiCF/03c5565060bb604d3f080f88f20309db 2024-12-03T15:57:21,140 DEBUG [RS_OPEN_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1048): stopping wal replay for 03c5565060bb604d3f080f88f20309db 2024-12-03T15:57:21,140 DEBUG [RS_OPEN_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1060): Cleaning up temporary data for 03c5565060bb604d3f080f88f20309db 2024-12-03T15:57:21,140 DEBUG [RS_OPEN_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-12-03T15:57:21,142 DEBUG [RS_OPEN_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1093): writing seq id for 03c5565060bb604d3f080f88f20309db 2024-12-03T15:57:21,143 INFO [RS_OPEN_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1114): Opened 03c5565060bb604d3f080f88f20309db; next sequenceid=18; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72801736, jitterRate=0.0848304033279419}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-12-03T15:57:21,143 DEBUG [RS_OPEN_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 03c5565060bb604d3f080f88f20309db 2024-12-03T15:57:21,143 DEBUG [RS_OPEN_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1006): Region open journal for 03c5565060bb604d3f080f88f20309db: Running coprocessor pre-open hook at 1733241441113Writing region info on filesystem at 1733241441113Initializing all the Stores at 1733241441114 (+1 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733241441114Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733241441115 (+1 ms)Cleaning up temporary data from old regions at 1733241441140 (+25 ms)Running coprocessor post-open hooks at 1733241441143 (+3 ms)Region opened successfully at 1733241441143 2024-12-03T15:57:21,144 INFO [RS_OPEN_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1733241437409.03c5565060bb604d3f080f88f20309db., pid=17, masterSystemTime=1733241441107 2024-12-03T15:57:21,147 DEBUG [RS_OPEN_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1733241437409.03c5565060bb604d3f080f88f20309db. 2024-12-03T15:57:21,147 INFO [RS_OPEN_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1733241437409.03c5565060bb604d3f080f88f20309db. 2024-12-03T15:57:21,148 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=03c5565060bb604d3f080f88f20309db, regionState=OPEN, openSeqNum=18, regionLocation=713339a81dd9,42147,1733241418180 2024-12-03T15:57:21,150 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=17, ppid=16, state=RUNNABLE, hasLock=false; OpenRegionProcedure 03c5565060bb604d3f080f88f20309db, server=713339a81dd9,42147,1733241418180 because future has completed 2024-12-03T15:57:21,154 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=17, resume processing ppid=16 2024-12-03T15:57:21,154 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, ppid=16, state=SUCCESS, hasLock=false; OpenRegionProcedure 03c5565060bb604d3f080f88f20309db, server=713339a81dd9,42147,1733241418180 in 197 msec 2024-12-03T15:57:21,157 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=13 2024-12-03T15:57:21,157 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=13, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=03c5565060bb604d3f080f88f20309db, ASSIGN in 359 msec 2024-12-03T15:57:21,157 INFO [PEWorker-3 {}] procedure.ServerCrashProcedure(291): removed crashed server 713339a81dd9,36387,1733241418383 after splitting done 2024-12-03T15:57:21,163 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; ServerCrashProcedure 713339a81dd9,36387,1733241418383, splitWal=true, meta=false in 664 msec 2024-12-03T15:57:21,423 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1733241437409.03c5565060bb604d3f080f88f20309db., hostname=713339a81dd9,42147,1733241418180, seqNum=18] 2024-12-03T15:57:21,438 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsAfterRegionMovedWithMultiCF Thread=409 (was 415), OpenFileDescriptor=1049 (was 1023) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=855 (was 855), ProcessCount=11 (was 11), AvailableMemoryMB=4825 (was 4929) 2024-12-03T15:57:21,440 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1049 is superior to 1024 2024-12-03T15:57:21,454 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsAfterPartialFlush Thread=409, OpenFileDescriptor=1049, MaxFileDescriptor=1048576, SystemLoadAverage=855, ProcessCount=11, AvailableMemoryMB=4822 2024-12-03T15:57:21,454 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1049 is superior to 1024 2024-12-03T15:57:21,470 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-03T15:57:21,472 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-03T15:57:21,473 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-03T15:57:21,478 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-68982327, suffix=, logDir=hdfs://localhost:45015/hbase/WALs/hregion-68982327, archiveDir=hdfs://localhost:45015/hbase/oldWALs, maxLogs=32 2024-12-03T15:57:21,496 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-68982327/hregion-68982327.1733241441478, exclude list is [], retry=0 2024-12-03T15:57:21,500 DEBUG [AsyncFSWAL-20-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40575,DS-309fe9b7-1997-44cb-b0c9-1ce3c04a71de,DISK] 2024-12-03T15:57:21,501 DEBUG [AsyncFSWAL-20-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44827,DS-e007f1cc-fc1b-4368-9716-e11e4fe2f4eb,DISK] 2024-12-03T15:57:21,502 DEBUG [AsyncFSWAL-20-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46585,DS-5c6771fd-75c7-4775-ad88-ae5bbe88de5c,DISK] 2024-12-03T15:57:21,535 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-68982327/hregion-68982327.1733241441478 2024-12-03T15:57:21,535 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44513:44513),(127.0.0.1/127.0.0.1:37355:37355),(127.0.0.1/127.0.0.1:46731:46731)] 2024-12-03T15:57:21,535 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 154b277e0cfe0796c0a278867a0045a9, NAME => 'testReplayEditsWrittenViaHRegion,,1733241441471.154b277e0cfe0796c0a278867a0045a9.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenViaHRegion', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45015/hbase 2024-12-03T15:57:21,545 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46585 is added to blk_1073741893_1071 (size=67) 2024-12-03T15:57:21,545 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44827 is added to blk_1073741893_1071 (size=67) 2024-12-03T15:57:21,545 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40575 is added to blk_1073741893_1071 (size=67) 2024-12-03T15:57:21,546 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1733241441471.154b277e0cfe0796c0a278867a0045a9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T15:57:21,548 INFO [StoreOpener-154b277e0cfe0796c0a278867a0045a9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 154b277e0cfe0796c0a278867a0045a9 2024-12-03T15:57:21,549 INFO [StoreOpener-154b277e0cfe0796c0a278867a0045a9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 154b277e0cfe0796c0a278867a0045a9 columnFamilyName a 2024-12-03T15:57:21,550 DEBUG [StoreOpener-154b277e0cfe0796c0a278867a0045a9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T15:57:21,550 INFO [StoreOpener-154b277e0cfe0796c0a278867a0045a9-1 {}] regionserver.HStore(327): Store=154b277e0cfe0796c0a278867a0045a9/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T15:57:21,550 INFO [StoreOpener-154b277e0cfe0796c0a278867a0045a9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 154b277e0cfe0796c0a278867a0045a9 2024-12-03T15:57:21,552 INFO [StoreOpener-154b277e0cfe0796c0a278867a0045a9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 154b277e0cfe0796c0a278867a0045a9 columnFamilyName b 2024-12-03T15:57:21,552 DEBUG [StoreOpener-154b277e0cfe0796c0a278867a0045a9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T15:57:21,553 INFO [StoreOpener-154b277e0cfe0796c0a278867a0045a9-1 {}] regionserver.HStore(327): Store=154b277e0cfe0796c0a278867a0045a9/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T15:57:21,553 INFO [StoreOpener-154b277e0cfe0796c0a278867a0045a9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 154b277e0cfe0796c0a278867a0045a9 2024-12-03T15:57:21,554 INFO [StoreOpener-154b277e0cfe0796c0a278867a0045a9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 154b277e0cfe0796c0a278867a0045a9 columnFamilyName c 2024-12-03T15:57:21,554 DEBUG [StoreOpener-154b277e0cfe0796c0a278867a0045a9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T15:57:21,555 INFO [StoreOpener-154b277e0cfe0796c0a278867a0045a9-1 {}] regionserver.HStore(327): Store=154b277e0cfe0796c0a278867a0045a9/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T15:57:21,555 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 154b277e0cfe0796c0a278867a0045a9 2024-12-03T15:57:21,556 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45015/hbase/data/default/testReplayEditsWrittenViaHRegion/154b277e0cfe0796c0a278867a0045a9 2024-12-03T15:57:21,556 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45015/hbase/data/default/testReplayEditsWrittenViaHRegion/154b277e0cfe0796c0a278867a0045a9 2024-12-03T15:57:21,558 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 154b277e0cfe0796c0a278867a0045a9 2024-12-03T15:57:21,558 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 154b277e0cfe0796c0a278867a0045a9 2024-12-03T15:57:21,558 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-03T15:57:21,559 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 154b277e0cfe0796c0a278867a0045a9 2024-12-03T15:57:21,562 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45015/hbase/data/default/testReplayEditsWrittenViaHRegion/154b277e0cfe0796c0a278867a0045a9/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T15:57:21,562 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 154b277e0cfe0796c0a278867a0045a9; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72371890, jitterRate=0.07842519879341125}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-03T15:57:21,563 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 154b277e0cfe0796c0a278867a0045a9: Writing region info on filesystem at 1733241441546Initializing all the Stores at 1733241441547 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733241441547Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733241441547Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733241441547Cleaning up temporary data from old regions at 1733241441558 (+11 ms)Region opened successfully at 1733241441563 (+5 ms) 2024-12-03T15:57:21,563 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 154b277e0cfe0796c0a278867a0045a9, disabling compactions & flushes 2024-12-03T15:57:21,563 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1733241441471.154b277e0cfe0796c0a278867a0045a9. 2024-12-03T15:57:21,563 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1733241441471.154b277e0cfe0796c0a278867a0045a9. 2024-12-03T15:57:21,563 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1733241441471.154b277e0cfe0796c0a278867a0045a9. after waiting 0 ms 2024-12-03T15:57:21,563 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1733241441471.154b277e0cfe0796c0a278867a0045a9. 2024-12-03T15:57:21,564 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1733241441471.154b277e0cfe0796c0a278867a0045a9. 2024-12-03T15:57:21,564 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 154b277e0cfe0796c0a278867a0045a9: Waiting for close lock at 1733241441563Disabling compacts and flushes for region at 1733241441563Disabling writes for close at 1733241441563Writing region close event to WAL at 1733241441564 (+1 ms)Closed at 1733241441564 2024-12-03T15:57:21,567 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40575 is added to blk_1073741892_1070 (size=175) 2024-12-03T15:57:21,567 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46585 is added to blk_1073741892_1070 (size=175) 2024-12-03T15:57:21,567 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44827 is added to blk_1073741892_1070 (size=175) 2024-12-03T15:57:21,570 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-03T15:57:21,570 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-68982327:(num 1733241441478) 2024-12-03T15:57:21,570 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-03T15:57:21,572 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:45015/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733241441470, archiveDir=hdfs://localhost:45015/hbase/oldWALs, maxLogs=32 2024-12-03T15:57:21,588 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733241441470/wal.1733241441572, exclude list is [], retry=0 2024-12-03T15:57:21,591 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40575,DS-309fe9b7-1997-44cb-b0c9-1ce3c04a71de,DISK] 2024-12-03T15:57:21,592 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44827,DS-e007f1cc-fc1b-4368-9716-e11e4fe2f4eb,DISK] 2024-12-03T15:57:21,592 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46585,DS-5c6771fd-75c7-4775-ad88-ae5bbe88de5c,DISK] 2024-12-03T15:57:21,628 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733241441470/wal.1733241441572 2024-12-03T15:57:21,628 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44513:44513),(127.0.0.1/127.0.0.1:46731:46731),(127.0.0.1/127.0.0.1:37355:37355)] 2024-12-03T15:57:21,628 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 154b277e0cfe0796c0a278867a0045a9, NAME => 'testReplayEditsWrittenViaHRegion,,1733241441471.154b277e0cfe0796c0a278867a0045a9.', STARTKEY => '', ENDKEY => ''} 2024-12-03T15:57:21,629 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1733241441471.154b277e0cfe0796c0a278867a0045a9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T15:57:21,629 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 154b277e0cfe0796c0a278867a0045a9 2024-12-03T15:57:21,629 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 154b277e0cfe0796c0a278867a0045a9 2024-12-03T15:57:21,631 INFO [StoreOpener-154b277e0cfe0796c0a278867a0045a9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 154b277e0cfe0796c0a278867a0045a9 2024-12-03T15:57:21,632 INFO [StoreOpener-154b277e0cfe0796c0a278867a0045a9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 154b277e0cfe0796c0a278867a0045a9 columnFamilyName a 2024-12-03T15:57:21,632 DEBUG [StoreOpener-154b277e0cfe0796c0a278867a0045a9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T15:57:21,632 INFO [StoreOpener-154b277e0cfe0796c0a278867a0045a9-1 {}] regionserver.HStore(327): Store=154b277e0cfe0796c0a278867a0045a9/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T15:57:21,633 INFO [StoreOpener-154b277e0cfe0796c0a278867a0045a9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 154b277e0cfe0796c0a278867a0045a9 2024-12-03T15:57:21,634 INFO [StoreOpener-154b277e0cfe0796c0a278867a0045a9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 154b277e0cfe0796c0a278867a0045a9 columnFamilyName b 2024-12-03T15:57:21,634 DEBUG [StoreOpener-154b277e0cfe0796c0a278867a0045a9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T15:57:21,634 INFO [StoreOpener-154b277e0cfe0796c0a278867a0045a9-1 {}] regionserver.HStore(327): Store=154b277e0cfe0796c0a278867a0045a9/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T15:57:21,634 INFO [StoreOpener-154b277e0cfe0796c0a278867a0045a9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 154b277e0cfe0796c0a278867a0045a9 2024-12-03T15:57:21,635 INFO [StoreOpener-154b277e0cfe0796c0a278867a0045a9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 154b277e0cfe0796c0a278867a0045a9 columnFamilyName c 2024-12-03T15:57:21,635 DEBUG [StoreOpener-154b277e0cfe0796c0a278867a0045a9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T15:57:21,636 INFO [StoreOpener-154b277e0cfe0796c0a278867a0045a9-1 {}] regionserver.HStore(327): Store=154b277e0cfe0796c0a278867a0045a9/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T15:57:21,636 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 154b277e0cfe0796c0a278867a0045a9 2024-12-03T15:57:21,637 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45015/hbase/data/default/testReplayEditsWrittenViaHRegion/154b277e0cfe0796c0a278867a0045a9 2024-12-03T15:57:21,639 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45015/hbase/data/default/testReplayEditsWrittenViaHRegion/154b277e0cfe0796c0a278867a0045a9 2024-12-03T15:57:21,640 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 154b277e0cfe0796c0a278867a0045a9 2024-12-03T15:57:21,640 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 154b277e0cfe0796c0a278867a0045a9 2024-12-03T15:57:21,641 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-03T15:57:21,642 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 154b277e0cfe0796c0a278867a0045a9 2024-12-03T15:57:21,643 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 154b277e0cfe0796c0a278867a0045a9; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67006716, jitterRate=-0.0015221238136291504}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-03T15:57:21,644 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 154b277e0cfe0796c0a278867a0045a9: Writing region info on filesystem at 1733241441629Initializing all the Stores at 1733241441630 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733241441630Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733241441630Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733241441630Cleaning up temporary data from old regions at 1733241441640 (+10 ms)Region opened successfully at 1733241441644 (+4 ms) 2024-12-03T15:57:21,714 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 154b277e0cfe0796c0a278867a0045a9 3/3 column families, dataSize=2.55 KB heapSize=5.44 KB 2024-12-03T15:57:21,739 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45015/hbase/data/default/testReplayEditsWrittenViaHRegion/154b277e0cfe0796c0a278867a0045a9/.tmp/a/bcc981301df34b4c8ba845d168dfe0b2 is 91, key is testReplayEditsWrittenViaHRegion/a:x0/1733241441644/Put/seqid=0 2024-12-03T15:57:21,749 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46585 is added to blk_1073741895_1073 (size=5958) 2024-12-03T15:57:21,750 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44827 is added to blk_1073741895_1073 (size=5958) 2024-12-03T15:57:21,750 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40575 is added to blk_1073741895_1073 (size=5958) 2024-12-03T15:57:21,759 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:45015/hbase/data/default/testReplayEditsWrittenViaHRegion/154b277e0cfe0796c0a278867a0045a9/.tmp/a/bcc981301df34b4c8ba845d168dfe0b2 2024-12-03T15:57:21,809 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45015/hbase/data/default/testReplayEditsWrittenViaHRegion/154b277e0cfe0796c0a278867a0045a9/.tmp/b/6f5a5f45b7bc40e286c0a5f9624e37ee is 91, key is testReplayEditsWrittenViaHRegion/b:x0/1733241441663/Put/seqid=0 2024-12-03T15:57:21,820 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46585 is added to blk_1073741896_1074 (size=5958) 2024-12-03T15:57:21,822 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40575 is added to blk_1073741896_1074 (size=5958) 2024-12-03T15:57:21,823 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44827 is added to blk_1073741896_1074 (size=5958) 2024-12-03T15:57:21,823 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:45015/hbase/data/default/testReplayEditsWrittenViaHRegion/154b277e0cfe0796c0a278867a0045a9/.tmp/b/6f5a5f45b7bc40e286c0a5f9624e37ee 2024-12-03T15:57:21,850 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45015/hbase/data/default/testReplayEditsWrittenViaHRegion/154b277e0cfe0796c0a278867a0045a9/.tmp/c/2f062be6bc8a480c8b00b9a2cd928f06 is 91, key is testReplayEditsWrittenViaHRegion/c:x0/1733241441694/Put/seqid=0 2024-12-03T15:57:21,857 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46585 is added to blk_1073741897_1075 (size=5958) 2024-12-03T15:57:21,858 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40575 is added to blk_1073741897_1075 (size=5958) 2024-12-03T15:57:21,858 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44827 is added to blk_1073741897_1075 (size=5958) 2024-12-03T15:57:21,859 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:45015/hbase/data/default/testReplayEditsWrittenViaHRegion/154b277e0cfe0796c0a278867a0045a9/.tmp/c/2f062be6bc8a480c8b00b9a2cd928f06 2024-12-03T15:57:21,866 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45015/hbase/data/default/testReplayEditsWrittenViaHRegion/154b277e0cfe0796c0a278867a0045a9/.tmp/a/bcc981301df34b4c8ba845d168dfe0b2 as hdfs://localhost:45015/hbase/data/default/testReplayEditsWrittenViaHRegion/154b277e0cfe0796c0a278867a0045a9/a/bcc981301df34b4c8ba845d168dfe0b2 2024-12-03T15:57:21,873 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45015/hbase/data/default/testReplayEditsWrittenViaHRegion/154b277e0cfe0796c0a278867a0045a9/a/bcc981301df34b4c8ba845d168dfe0b2, entries=10, sequenceid=33, filesize=5.8 K 2024-12-03T15:57:21,874 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45015/hbase/data/default/testReplayEditsWrittenViaHRegion/154b277e0cfe0796c0a278867a0045a9/.tmp/b/6f5a5f45b7bc40e286c0a5f9624e37ee as hdfs://localhost:45015/hbase/data/default/testReplayEditsWrittenViaHRegion/154b277e0cfe0796c0a278867a0045a9/b/6f5a5f45b7bc40e286c0a5f9624e37ee 2024-12-03T15:57:21,880 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45015/hbase/data/default/testReplayEditsWrittenViaHRegion/154b277e0cfe0796c0a278867a0045a9/b/6f5a5f45b7bc40e286c0a5f9624e37ee, entries=10, sequenceid=33, filesize=5.8 K 2024-12-03T15:57:21,882 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45015/hbase/data/default/testReplayEditsWrittenViaHRegion/154b277e0cfe0796c0a278867a0045a9/.tmp/c/2f062be6bc8a480c8b00b9a2cd928f06 as hdfs://localhost:45015/hbase/data/default/testReplayEditsWrittenViaHRegion/154b277e0cfe0796c0a278867a0045a9/c/2f062be6bc8a480c8b00b9a2cd928f06 2024-12-03T15:57:21,888 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45015/hbase/data/default/testReplayEditsWrittenViaHRegion/154b277e0cfe0796c0a278867a0045a9/c/2f062be6bc8a480c8b00b9a2cd928f06, entries=10, sequenceid=33, filesize=5.8 K 2024-12-03T15:57:21,889 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~2.55 KB/2610, heapSize ~5.39 KB/5520, currentSize=0 B/0 for 154b277e0cfe0796c0a278867a0045a9 in 175ms, sequenceid=33, compaction requested=false 2024-12-03T15:57:21,889 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 154b277e0cfe0796c0a278867a0045a9: 2024-12-03T15:57:21,889 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 154b277e0cfe0796c0a278867a0045a9, disabling compactions & flushes 2024-12-03T15:57:21,889 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1733241441471.154b277e0cfe0796c0a278867a0045a9. 2024-12-03T15:57:21,890 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1733241441471.154b277e0cfe0796c0a278867a0045a9. 2024-12-03T15:57:21,890 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1733241441471.154b277e0cfe0796c0a278867a0045a9. after waiting 0 ms 2024-12-03T15:57:21,890 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1733241441471.154b277e0cfe0796c0a278867a0045a9. 2024-12-03T15:57:21,894 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1733241441471.154b277e0cfe0796c0a278867a0045a9. 2024-12-03T15:57:21,894 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 154b277e0cfe0796c0a278867a0045a9: Waiting for close lock at 1733241441889Disabling compacts and flushes for region at 1733241441889Disabling writes for close at 1733241441890 (+1 ms)Writing region close event to WAL at 1733241441894 (+4 ms)Closed at 1733241441894 2024-12-03T15:57:21,897 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46585 is added to blk_1073741894_1072 (size=6311) 2024-12-03T15:57:21,897 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40575 is added to blk_1073741894_1072 (size=6311) 2024-12-03T15:57:21,897 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44827 is added to blk_1073741894_1072 (size=6311) 2024-12-03T15:57:21,902 DEBUG [Time-limited test {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45015/hbase/data/default/testReplayEditsWrittenViaHRegion/154b277e0cfe0796c0a278867a0045a9/b/6f5a5f45b7bc40e286c0a5f9624e37ee to hdfs://localhost:45015/hbase/archive/data/default/testReplayEditsWrittenViaHRegion/154b277e0cfe0796c0a278867a0045a9/b/6f5a5f45b7bc40e286c0a5f9624e37ee 2024-12-03T15:57:21,920 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:45015/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733241441470/wal.1733241441572, size=6.2 K (6311bytes) 2024-12-03T15:57:21,920 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:45015/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733241441470/wal.1733241441572 2024-12-03T15:57:21,920 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:45015/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733241441470/wal.1733241441572 after 0ms 2024-12-03T15:57:21,949 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-03T15:57:21,950 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:45015/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733241441470/wal.1733241441572 took 29ms 2024-12-03T15:57:21,953 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:45015/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733241441470/wal.1733241441572 so closing down 2024-12-03T15:57:21,953 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-03T15:57:21,954 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1733241441572.temp 2024-12-03T15:57:21,981 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:45015/hbase/data/default/testReplayEditsWrittenViaHRegion/154b277e0cfe0796c0a278867a0045a9/recovered.edits/0000000000000000003-wal.1733241441572.temp 2024-12-03T15:57:21,981 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-03T15:57:21,995 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40575 is added to blk_1073741898_1076 (size=5575) 2024-12-03T15:57:21,995 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46585 is added to blk_1073741898_1076 (size=5575) 2024-12-03T15:57:21,995 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44827 is added to blk_1073741898_1076 (size=5575) 2024-12-03T15:57:21,996 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:45015/hbase/data/default/testReplayEditsWrittenViaHRegion/154b277e0cfe0796c0a278867a0045a9/recovered.edits/0000000000000000003-wal.1733241441572.temp (wrote 30 edits, skipped 0 edits in 0 ms) 2024-12-03T15:57:21,998 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:45015/hbase/data/default/testReplayEditsWrittenViaHRegion/154b277e0cfe0796c0a278867a0045a9/recovered.edits/0000000000000000003-wal.1733241441572.temp to hdfs://localhost:45015/hbase/data/default/testReplayEditsWrittenViaHRegion/154b277e0cfe0796c0a278867a0045a9/recovered.edits/0000000000000000032 2024-12-03T15:57:21,998 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 32 edits across 1 Regions in 48 ms; skipped=2; WAL=hdfs://localhost:45015/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733241441470/wal.1733241441572, size=6.2 K, length=6311, corrupted=false, cancelled=false 2024-12-03T15:57:21,998 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:45015/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733241441470/wal.1733241441572, journal: Splitting hdfs://localhost:45015/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733241441470/wal.1733241441572, size=6.2 K (6311bytes) at 1733241441920Finishing writing output for hdfs://localhost:45015/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733241441470/wal.1733241441572 so closing down at 1733241441953 (+33 ms)Creating recovered edits writer path=hdfs://localhost:45015/hbase/data/default/testReplayEditsWrittenViaHRegion/154b277e0cfe0796c0a278867a0045a9/recovered.edits/0000000000000000003-wal.1733241441572.temp at 1733241441981 (+28 ms)3 split writer threads finished at 1733241441981Closed recovered edits writer path=hdfs://localhost:45015/hbase/data/default/testReplayEditsWrittenViaHRegion/154b277e0cfe0796c0a278867a0045a9/recovered.edits/0000000000000000003-wal.1733241441572.temp (wrote 30 edits, skipped 0 edits in 0 ms) at 1733241441996 (+15 ms)Rename recovered edits hdfs://localhost:45015/hbase/data/default/testReplayEditsWrittenViaHRegion/154b277e0cfe0796c0a278867a0045a9/recovered.edits/0000000000000000003-wal.1733241441572.temp to hdfs://localhost:45015/hbase/data/default/testReplayEditsWrittenViaHRegion/154b277e0cfe0796c0a278867a0045a9/recovered.edits/0000000000000000032 at 1733241441998 (+2 ms)Processed 32 edits across 1 Regions in 48 ms; skipped=2; WAL=hdfs://localhost:45015/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733241441470/wal.1733241441572, size=6.2 K, length=6311, corrupted=false, cancelled=false at 1733241441998 2024-12-03T15:57:22,011 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:45015/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733241441470/wal.1733241441572 to hdfs://localhost:45015/hbase/oldWALs/wal.1733241441572 2024-12-03T15:57:22,012 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:45015/hbase/data/default/testReplayEditsWrittenViaHRegion/154b277e0cfe0796c0a278867a0045a9/recovered.edits/0000000000000000032 2024-12-03T15:57:22,012 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-03T15:57:22,014 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:45015/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733241441470, archiveDir=hdfs://localhost:45015/hbase/oldWALs, maxLogs=32 2024-12-03T15:57:22,035 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733241441470/wal.1733241442014, exclude list is [], retry=0 2024-12-03T15:57:22,039 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46585,DS-5c6771fd-75c7-4775-ad88-ae5bbe88de5c,DISK] 2024-12-03T15:57:22,039 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44827,DS-e007f1cc-fc1b-4368-9716-e11e4fe2f4eb,DISK] 2024-12-03T15:57:22,039 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40575,DS-309fe9b7-1997-44cb-b0c9-1ce3c04a71de,DISK] 2024-12-03T15:57:22,069 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733241441470/wal.1733241442014 2024-12-03T15:57:22,069 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37355:37355),(127.0.0.1/127.0.0.1:46731:46731),(127.0.0.1/127.0.0.1:44513:44513)] 2024-12-03T15:57:22,069 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 154b277e0cfe0796c0a278867a0045a9, NAME => 'testReplayEditsWrittenViaHRegion,,1733241441471.154b277e0cfe0796c0a278867a0045a9.', STARTKEY => '', ENDKEY => ''} 2024-12-03T15:57:22,070 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1733241441471.154b277e0cfe0796c0a278867a0045a9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T15:57:22,070 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 154b277e0cfe0796c0a278867a0045a9 2024-12-03T15:57:22,070 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 154b277e0cfe0796c0a278867a0045a9 2024-12-03T15:57:22,072 INFO [StoreOpener-154b277e0cfe0796c0a278867a0045a9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 154b277e0cfe0796c0a278867a0045a9 2024-12-03T15:57:22,073 INFO [StoreOpener-154b277e0cfe0796c0a278867a0045a9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 154b277e0cfe0796c0a278867a0045a9 columnFamilyName a 2024-12-03T15:57:22,073 DEBUG [StoreOpener-154b277e0cfe0796c0a278867a0045a9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T15:57:22,079 DEBUG [StoreOpener-154b277e0cfe0796c0a278867a0045a9-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:45015/hbase/data/default/testReplayEditsWrittenViaHRegion/154b277e0cfe0796c0a278867a0045a9/a/bcc981301df34b4c8ba845d168dfe0b2 2024-12-03T15:57:22,079 INFO [StoreOpener-154b277e0cfe0796c0a278867a0045a9-1 {}] regionserver.HStore(327): Store=154b277e0cfe0796c0a278867a0045a9/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T15:57:22,079 INFO [StoreOpener-154b277e0cfe0796c0a278867a0045a9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 154b277e0cfe0796c0a278867a0045a9 2024-12-03T15:57:22,080 INFO [StoreOpener-154b277e0cfe0796c0a278867a0045a9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 154b277e0cfe0796c0a278867a0045a9 columnFamilyName b 2024-12-03T15:57:22,081 DEBUG [StoreOpener-154b277e0cfe0796c0a278867a0045a9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T15:57:22,081 INFO [StoreOpener-154b277e0cfe0796c0a278867a0045a9-1 {}] regionserver.HStore(327): Store=154b277e0cfe0796c0a278867a0045a9/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T15:57:22,081 INFO [StoreOpener-154b277e0cfe0796c0a278867a0045a9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 154b277e0cfe0796c0a278867a0045a9 2024-12-03T15:57:22,082 INFO [StoreOpener-154b277e0cfe0796c0a278867a0045a9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 154b277e0cfe0796c0a278867a0045a9 columnFamilyName c 2024-12-03T15:57:22,082 DEBUG [StoreOpener-154b277e0cfe0796c0a278867a0045a9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T15:57:22,089 DEBUG [StoreOpener-154b277e0cfe0796c0a278867a0045a9-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:45015/hbase/data/default/testReplayEditsWrittenViaHRegion/154b277e0cfe0796c0a278867a0045a9/c/2f062be6bc8a480c8b00b9a2cd928f06 2024-12-03T15:57:22,089 INFO [StoreOpener-154b277e0cfe0796c0a278867a0045a9-1 {}] regionserver.HStore(327): Store=154b277e0cfe0796c0a278867a0045a9/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T15:57:22,090 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 154b277e0cfe0796c0a278867a0045a9 2024-12-03T15:57:22,091 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45015/hbase/data/default/testReplayEditsWrittenViaHRegion/154b277e0cfe0796c0a278867a0045a9 2024-12-03T15:57:22,093 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:45015/hbase/data/default/testReplayEditsWrittenViaHRegion/154b277e0cfe0796c0a278867a0045a9 2024-12-03T15:57:22,094 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:45015/hbase/data/default/testReplayEditsWrittenViaHRegion/154b277e0cfe0796c0a278867a0045a9/recovered.edits/0000000000000000032 2024-12-03T15:57:22,131 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-03T15:57:22,133 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 10, skipped 20, firstSequenceIdInLog=3, maxSequenceIdInLog=32, path=hdfs://localhost:45015/hbase/data/default/testReplayEditsWrittenViaHRegion/154b277e0cfe0796c0a278867a0045a9/recovered.edits/0000000000000000032 2024-12-03T15:57:22,134 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 154b277e0cfe0796c0a278867a0045a9 3/3 column families, dataSize=870 B heapSize=2.31 KB 2024-12-03T15:57:22,157 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45015/hbase/data/default/testReplayEditsWrittenViaHRegion/154b277e0cfe0796c0a278867a0045a9/.tmp/b/86631f4fa5b34ae4b576185ccce04ecf is 91, key is testReplayEditsWrittenViaHRegion/b:x0/1733241441663/Put/seqid=0 2024-12-03T15:57:22,172 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40575 is added to blk_1073741900_1078 (size=5958) 2024-12-03T15:57:22,173 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46585 is added to blk_1073741900_1078 (size=5958) 2024-12-03T15:57:22,174 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44827 is added to blk_1073741900_1078 (size=5958) 2024-12-03T15:57:22,174 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=32 (bloomFilter=true), to=hdfs://localhost:45015/hbase/data/default/testReplayEditsWrittenViaHRegion/154b277e0cfe0796c0a278867a0045a9/.tmp/b/86631f4fa5b34ae4b576185ccce04ecf 2024-12-03T15:57:22,186 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45015/hbase/data/default/testReplayEditsWrittenViaHRegion/154b277e0cfe0796c0a278867a0045a9/.tmp/b/86631f4fa5b34ae4b576185ccce04ecf as hdfs://localhost:45015/hbase/data/default/testReplayEditsWrittenViaHRegion/154b277e0cfe0796c0a278867a0045a9/b/86631f4fa5b34ae4b576185ccce04ecf 2024-12-03T15:57:22,197 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45015/hbase/data/default/testReplayEditsWrittenViaHRegion/154b277e0cfe0796c0a278867a0045a9/b/86631f4fa5b34ae4b576185ccce04ecf, entries=10, sequenceid=32, filesize=5.8 K 2024-12-03T15:57:22,198 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~870 B/870, heapSize ~1.80 KB/1840, currentSize=0 B/0 for 154b277e0cfe0796c0a278867a0045a9 in 64ms, sequenceid=32, compaction requested=false; wal=null 2024-12-03T15:57:22,199 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:45015/hbase/data/default/testReplayEditsWrittenViaHRegion/154b277e0cfe0796c0a278867a0045a9/recovered.edits/0000000000000000032 2024-12-03T15:57:22,200 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 154b277e0cfe0796c0a278867a0045a9 2024-12-03T15:57:22,200 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 154b277e0cfe0796c0a278867a0045a9 2024-12-03T15:57:22,201 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-03T15:57:22,202 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 154b277e0cfe0796c0a278867a0045a9 2024-12-03T15:57:22,205 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45015/hbase/data/default/testReplayEditsWrittenViaHRegion/154b277e0cfe0796c0a278867a0045a9/recovered.edits/33.seqid, newMaxSeqId=33, maxSeqId=1 2024-12-03T15:57:22,207 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 154b277e0cfe0796c0a278867a0045a9; next sequenceid=34; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72281043, jitterRate=0.07707147300243378}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-03T15:57:22,207 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 154b277e0cfe0796c0a278867a0045a9: Writing region info on filesystem at 1733241442070Initializing all the Stores at 1733241442071 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733241442071Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733241442072 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733241442072Obtaining lock to block concurrent updates at 1733241442134 (+62 ms)Preparing flush snapshotting stores in 154b277e0cfe0796c0a278867a0045a9 at 1733241442134Finished memstore snapshotting testReplayEditsWrittenViaHRegion,,1733241441471.154b277e0cfe0796c0a278867a0045a9., syncing WAL and waiting on mvcc, flushsize=dataSize=870, getHeapSize=2320, getOffHeapSize=0, getCellsCount=10 at 1733241442134Flushing stores of testReplayEditsWrittenViaHRegion,,1733241441471.154b277e0cfe0796c0a278867a0045a9. at 1733241442134Flushing 154b277e0cfe0796c0a278867a0045a9/b: creating writer at 1733241442134Flushing 154b277e0cfe0796c0a278867a0045a9/b: appending metadata at 1733241442156 (+22 ms)Flushing 154b277e0cfe0796c0a278867a0045a9/b: closing flushed file at 1733241442156Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@cc74c34: reopening flushed file at 1733241442185 (+29 ms)Finished flush of dataSize ~870 B/870, heapSize ~1.80 KB/1840, currentSize=0 B/0 for 154b277e0cfe0796c0a278867a0045a9 in 64ms, sequenceid=32, compaction requested=false; wal=null at 1733241442198 (+13 ms)Cleaning up temporary data from old regions at 1733241442200 (+2 ms)Region opened successfully at 1733241442207 (+7 ms) 2024-12-03T15:57:22,253 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsAfterPartialFlush Thread=418 (was 409) Potentially hanging thread: PacketResponder: BP-1122350750-172.17.0.3-1733241413817:blk_1073741899_1077, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-365845159_22 at /127.0.0.1:58428 [Waiting for operation #39] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1122350750-172.17.0.3-1733241413817:blk_1073741899_1077, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-20-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-365845159_22 at /127.0.0.1:53898 [Receiving block BP-1122350750-172.17.0.3-1733241413817:blk_1073741899_1077] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-20-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1122350750-172.17.0.3-1733241413817:blk_1073741899_1077, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-365845159_22 at /127.0.0.1:56472 [Waiting for operation #36] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-365845159_22 at /127.0.0.1:41064 [Waiting for operation #27] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-365845159_22 at /127.0.0.1:57884 [Receiving block BP-1122350750-172.17.0.3-1733241413817:blk_1073741899_1077] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-20-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-365845159_22 at /127.0.0.1:58888 [Receiving block BP-1122350750-172.17.0.3-1733241413817:blk_1073741899_1077] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1119 (was 1049) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=827 (was 855), ProcessCount=11 (was 11), AvailableMemoryMB=4942 (was 4822) - AvailableMemoryMB LEAK? - 2024-12-03T15:57:22,253 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1119 is superior to 1024 2024-12-03T15:57:22,269 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsAfterAbortingFlush Thread=418, OpenFileDescriptor=1119, MaxFileDescriptor=1048576, SystemLoadAverage=827, ProcessCount=11, AvailableMemoryMB=4939 2024-12-03T15:57:22,269 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1119 is superior to 1024 2024-12-03T15:57:22,293 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-03T15:57:22,296 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-03T15:57:22,297 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-03T15:57:22,300 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-59778005, suffix=, logDir=hdfs://localhost:45015/hbase/WALs/hregion-59778005, archiveDir=hdfs://localhost:45015/hbase/oldWALs, maxLogs=32 2024-12-03T15:57:22,315 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-59778005/hregion-59778005.1733241442300, exclude list is [], retry=0 2024-12-03T15:57:22,318 DEBUG [AsyncFSWAL-22-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44827,DS-e007f1cc-fc1b-4368-9716-e11e4fe2f4eb,DISK] 2024-12-03T15:57:22,319 DEBUG [AsyncFSWAL-22-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40575,DS-309fe9b7-1997-44cb-b0c9-1ce3c04a71de,DISK] 2024-12-03T15:57:22,322 DEBUG [AsyncFSWAL-22-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46585,DS-5c6771fd-75c7-4775-ad88-ae5bbe88de5c,DISK] 2024-12-03T15:57:22,355 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-59778005/hregion-59778005.1733241442300 2024-12-03T15:57:22,356 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46731:46731),(127.0.0.1/127.0.0.1:44513:44513),(127.0.0.1/127.0.0.1:37355:37355)] 2024-12-03T15:57:22,356 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => a0119fafb01eead8a6376bf521f59012, NAME => 'testReplayEditsAfterAbortingFlush,,1733241442294.a0119fafb01eead8a6376bf521f59012.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsAfterAbortingFlush', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45015/hbase 2024-12-03T15:57:22,372 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46585 is added to blk_1073741902_1080 (size=68) 2024-12-03T15:57:22,373 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44827 is added to blk_1073741902_1080 (size=68) 2024-12-03T15:57:22,373 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40575 is added to blk_1073741902_1080 (size=68) 2024-12-03T15:57:22,374 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterAbortingFlush,,1733241442294.a0119fafb01eead8a6376bf521f59012.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T15:57:22,376 INFO [StoreOpener-a0119fafb01eead8a6376bf521f59012-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region a0119fafb01eead8a6376bf521f59012 2024-12-03T15:57:22,377 INFO [StoreOpener-a0119fafb01eead8a6376bf521f59012-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a0119fafb01eead8a6376bf521f59012 columnFamilyName a 2024-12-03T15:57:22,377 DEBUG [StoreOpener-a0119fafb01eead8a6376bf521f59012-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T15:57:22,378 INFO [StoreOpener-a0119fafb01eead8a6376bf521f59012-1 {}] regionserver.HStore(327): Store=a0119fafb01eead8a6376bf521f59012/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T15:57:22,378 INFO [StoreOpener-a0119fafb01eead8a6376bf521f59012-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region a0119fafb01eead8a6376bf521f59012 2024-12-03T15:57:22,379 INFO [StoreOpener-a0119fafb01eead8a6376bf521f59012-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a0119fafb01eead8a6376bf521f59012 columnFamilyName b 2024-12-03T15:57:22,379 DEBUG [StoreOpener-a0119fafb01eead8a6376bf521f59012-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T15:57:22,380 INFO [StoreOpener-a0119fafb01eead8a6376bf521f59012-1 {}] regionserver.HStore(327): Store=a0119fafb01eead8a6376bf521f59012/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T15:57:22,380 INFO [StoreOpener-a0119fafb01eead8a6376bf521f59012-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region a0119fafb01eead8a6376bf521f59012 2024-12-03T15:57:22,381 INFO [StoreOpener-a0119fafb01eead8a6376bf521f59012-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a0119fafb01eead8a6376bf521f59012 columnFamilyName c 2024-12-03T15:57:22,381 DEBUG [StoreOpener-a0119fafb01eead8a6376bf521f59012-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T15:57:22,381 INFO [StoreOpener-a0119fafb01eead8a6376bf521f59012-1 {}] regionserver.HStore(327): Store=a0119fafb01eead8a6376bf521f59012/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T15:57:22,382 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for a0119fafb01eead8a6376bf521f59012 2024-12-03T15:57:22,382 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45015/hbase/data/default/testReplayEditsAfterAbortingFlush/a0119fafb01eead8a6376bf521f59012 2024-12-03T15:57:22,383 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45015/hbase/data/default/testReplayEditsAfterAbortingFlush/a0119fafb01eead8a6376bf521f59012 2024-12-03T15:57:22,384 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for a0119fafb01eead8a6376bf521f59012 2024-12-03T15:57:22,384 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for a0119fafb01eead8a6376bf521f59012 2024-12-03T15:57:22,384 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterAbortingFlush descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-03T15:57:22,386 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for a0119fafb01eead8a6376bf521f59012 2024-12-03T15:57:22,391 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45015/hbase/data/default/testReplayEditsAfterAbortingFlush/a0119fafb01eead8a6376bf521f59012/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T15:57:22,392 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened a0119fafb01eead8a6376bf521f59012; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60216983, jitterRate=-0.10269702970981598}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-03T15:57:22,393 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for a0119fafb01eead8a6376bf521f59012: Writing region info on filesystem at 1733241442374Initializing all the Stores at 1733241442375 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733241442375Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733241442375Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733241442375Cleaning up temporary data from old regions at 1733241442384 (+9 ms)Region opened successfully at 1733241442393 (+9 ms) 2024-12-03T15:57:22,393 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing a0119fafb01eead8a6376bf521f59012, disabling compactions & flushes 2024-12-03T15:57:22,393 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterAbortingFlush,,1733241442294.a0119fafb01eead8a6376bf521f59012. 2024-12-03T15:57:22,393 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterAbortingFlush,,1733241442294.a0119fafb01eead8a6376bf521f59012. 2024-12-03T15:57:22,393 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterAbortingFlush,,1733241442294.a0119fafb01eead8a6376bf521f59012. after waiting 0 ms 2024-12-03T15:57:22,393 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterAbortingFlush,,1733241442294.a0119fafb01eead8a6376bf521f59012. 2024-12-03T15:57:22,394 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsAfterAbortingFlush,,1733241442294.a0119fafb01eead8a6376bf521f59012. 2024-12-03T15:57:22,394 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for a0119fafb01eead8a6376bf521f59012: Waiting for close lock at 1733241442393Disabling compacts and flushes for region at 1733241442393Disabling writes for close at 1733241442393Writing region close event to WAL at 1733241442394 (+1 ms)Closed at 1733241442394 2024-12-03T15:57:22,401 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46585 is added to blk_1073741901_1079 (size=175) 2024-12-03T15:57:22,401 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44827 is added to blk_1073741901_1079 (size=175) 2024-12-03T15:57:22,401 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40575 is added to blk_1073741901_1079 (size=175) 2024-12-03T15:57:22,406 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-03T15:57:22,407 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-59778005:(num 1733241442300) 2024-12-03T15:57:22,407 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-03T15:57:22,409 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:45015/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733241442291, archiveDir=hdfs://localhost:45015/hbase/oldWALs, maxLogs=32 2024-12-03T15:57:22,428 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733241442291/wal.1733241442409, exclude list is [], retry=0 2024-12-03T15:57:22,432 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46585,DS-5c6771fd-75c7-4775-ad88-ae5bbe88de5c,DISK] 2024-12-03T15:57:22,432 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40575,DS-309fe9b7-1997-44cb-b0c9-1ce3c04a71de,DISK] 2024-12-03T15:57:22,433 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44827,DS-e007f1cc-fc1b-4368-9716-e11e4fe2f4eb,DISK] 2024-12-03T15:57:22,478 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733241442291/wal.1733241442409 2024-12-03T15:57:22,479 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37355:37355),(127.0.0.1/127.0.0.1:44513:44513),(127.0.0.1/127.0.0.1:46731:46731)] 2024-12-03T15:57:22,566 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => a0119fafb01eead8a6376bf521f59012, NAME => 'testReplayEditsAfterAbortingFlush,,1733241442294.a0119fafb01eead8a6376bf521f59012.', STARTKEY => '', ENDKEY => ''} 2024-12-03T15:57:22,570 DEBUG [Time-limited test {}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterAbortingFlush a0119fafb01eead8a6376bf521f59012 2024-12-03T15:57:22,570 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterAbortingFlush,,1733241442294.a0119fafb01eead8a6376bf521f59012.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T15:57:22,570 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for a0119fafb01eead8a6376bf521f59012 2024-12-03T15:57:22,570 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for a0119fafb01eead8a6376bf521f59012 2024-12-03T15:57:22,581 INFO [StoreOpener-a0119fafb01eead8a6376bf521f59012-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region a0119fafb01eead8a6376bf521f59012 2024-12-03T15:57:22,583 INFO [StoreOpener-a0119fafb01eead8a6376bf521f59012-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a0119fafb01eead8a6376bf521f59012 columnFamilyName a 2024-12-03T15:57:22,583 DEBUG [StoreOpener-a0119fafb01eead8a6376bf521f59012-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T15:57:22,584 INFO [StoreOpener-a0119fafb01eead8a6376bf521f59012-1 {}] regionserver.HStore(327): Store=a0119fafb01eead8a6376bf521f59012/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T15:57:22,584 INFO [StoreOpener-a0119fafb01eead8a6376bf521f59012-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region a0119fafb01eead8a6376bf521f59012 2024-12-03T15:57:22,585 INFO [StoreOpener-a0119fafb01eead8a6376bf521f59012-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a0119fafb01eead8a6376bf521f59012 columnFamilyName b 2024-12-03T15:57:22,585 DEBUG [StoreOpener-a0119fafb01eead8a6376bf521f59012-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T15:57:22,586 INFO [StoreOpener-a0119fafb01eead8a6376bf521f59012-1 {}] regionserver.HStore(327): Store=a0119fafb01eead8a6376bf521f59012/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T15:57:22,586 INFO [StoreOpener-a0119fafb01eead8a6376bf521f59012-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region a0119fafb01eead8a6376bf521f59012 2024-12-03T15:57:22,587 INFO [StoreOpener-a0119fafb01eead8a6376bf521f59012-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a0119fafb01eead8a6376bf521f59012 columnFamilyName c 2024-12-03T15:57:22,587 DEBUG [StoreOpener-a0119fafb01eead8a6376bf521f59012-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T15:57:22,588 INFO [StoreOpener-a0119fafb01eead8a6376bf521f59012-1 {}] regionserver.HStore(327): Store=a0119fafb01eead8a6376bf521f59012/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T15:57:22,588 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for a0119fafb01eead8a6376bf521f59012 2024-12-03T15:57:22,589 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45015/hbase/data/default/testReplayEditsAfterAbortingFlush/a0119fafb01eead8a6376bf521f59012 2024-12-03T15:57:22,591 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45015/hbase/data/default/testReplayEditsAfterAbortingFlush/a0119fafb01eead8a6376bf521f59012 2024-12-03T15:57:22,592 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for a0119fafb01eead8a6376bf521f59012 2024-12-03T15:57:22,592 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for a0119fafb01eead8a6376bf521f59012 2024-12-03T15:57:22,593 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterAbortingFlush descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-03T15:57:22,596 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for a0119fafb01eead8a6376bf521f59012 2024-12-03T15:57:22,599 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened a0119fafb01eead8a6376bf521f59012; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73528156, jitterRate=0.09565490484237671}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-03T15:57:22,599 DEBUG [Time-limited test {}] regionserver.HRegion(1122): Running coprocessor post-open hooks for a0119fafb01eead8a6376bf521f59012 2024-12-03T15:57:22,599 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for a0119fafb01eead8a6376bf521f59012: Running coprocessor pre-open hook at 1733241442570Writing region info on filesystem at 1733241442570Initializing all the Stores at 1733241442575 (+5 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733241442575Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733241442577 (+2 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733241442577Cleaning up temporary data from old regions at 1733241442592 (+15 ms)Running coprocessor post-open hooks at 1733241442599 (+7 ms)Region opened successfully at 1733241442599 2024-12-03T15:57:22,617 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing a0119fafb01eead8a6376bf521f59012 3/3 column families, dataSize=590 B heapSize=2.08 KB 2024-12-03T15:57:22,618 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for a0119fafb01eead8a6376bf521f59012/a, retrying num=0 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T15:57:23,382 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-03T15:57:23,619 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for a0119fafb01eead8a6376bf521f59012/a, retrying num=1 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T15:57:24,620 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for a0119fafb01eead8a6376bf521f59012/a, retrying num=2 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T15:57:25,621 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for a0119fafb01eead8a6376bf521f59012/a, retrying num=3 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T15:57:26,281 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-03T15:57:26,622 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for a0119fafb01eead8a6376bf521f59012/a, retrying num=4 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T15:57:27,623 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for a0119fafb01eead8a6376bf521f59012/a, retrying num=5 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T15:57:27,779 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenViaHRegion 2024-12-03T15:57:27,779 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenViaHRegion Metrics about Tables on a single HBase RegionServer 2024-12-03T15:57:27,780 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterRegionMovedWithMultiCF 2024-12-03T15:57:27,780 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterRegionMovedWithMultiCF Metrics about Tables on a single HBase RegionServer 2024-12-03T15:57:27,781 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterAbortingFlush 2024-12-03T15:57:27,781 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterAbortingFlush Metrics about Tables on a single HBase RegionServer 2024-12-03T15:57:28,623 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for a0119fafb01eead8a6376bf521f59012/a, retrying num=6 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T15:57:29,624 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for a0119fafb01eead8a6376bf521f59012/a, retrying num=7 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T15:57:30,625 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for a0119fafb01eead8a6376bf521f59012/a, retrying num=8 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T15:57:31,626 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for a0119fafb01eead8a6376bf521f59012/a, retrying num=9 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T15:57:31,627 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for a0119fafb01eead8a6376bf521f59012: 2024-12-03T15:57:31,627 INFO [Time-limited test {}] wal.AbstractTestWALReplay(671): Expected simulated exception when flushing region, region: testReplayEditsAfterAbortingFlush,,1733241442294.a0119fafb01eead8a6376bf521f59012. 2024-12-03T15:57:31,646 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for a0119fafb01eead8a6376bf521f59012: 2024-12-03T15:57:31,646 INFO [Time-limited test {}] wal.AbstractTestWALReplay(691): Expected exception when flushing region because server is stopped,Aborting flush because server is aborted... 2024-12-03T15:57:31,646 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing a0119fafb01eead8a6376bf521f59012, disabling compactions & flushes 2024-12-03T15:57:31,646 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterAbortingFlush,,1733241442294.a0119fafb01eead8a6376bf521f59012. 2024-12-03T15:57:31,646 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterAbortingFlush,,1733241442294.a0119fafb01eead8a6376bf521f59012. 2024-12-03T15:57:31,646 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterAbortingFlush,,1733241442294.a0119fafb01eead8a6376bf521f59012. after waiting 0 ms 2024-12-03T15:57:31,647 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterAbortingFlush,,1733241442294.a0119fafb01eead8a6376bf521f59012. 2024-12-03T15:57:31,647 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 1190 in region testReplayEditsAfterAbortingFlush,,1733241442294.a0119fafb01eead8a6376bf521f59012. 2024-12-03T15:57:31,647 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsAfterAbortingFlush,,1733241442294.a0119fafb01eead8a6376bf521f59012. 2024-12-03T15:57:31,647 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for a0119fafb01eead8a6376bf521f59012: Waiting for close lock at 1733241451646Running coprocessor pre-close hooks at 1733241451646Disabling compacts and flushes for region at 1733241451646Disabling writes for close at 1733241451646Writing region close event to WAL at 1733241451647 (+1 ms)Running coprocessor post-close hooks at 1733241451647Closed at 1733241451647 2024-12-03T15:57:31,654 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44827 is added to blk_1073741903_1081 (size=4236) 2024-12-03T15:57:31,655 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46585 is added to blk_1073741903_1081 (size=4236) 2024-12-03T15:57:31,655 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40575 is added to blk_1073741903_1081 (size=4236) 2024-12-03T15:57:31,655 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733241442291/wal.1733241442409 not finished, retry = 0 2024-12-03T15:57:31,778 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:45015/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733241442291/wal.1733241442409, size=4.1 K (4236bytes) 2024-12-03T15:57:31,778 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:45015/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733241442291/wal.1733241442409 2024-12-03T15:57:31,778 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:45015/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733241442291/wal.1733241442409 after 0ms 2024-12-03T15:57:31,820 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-03T15:57:31,820 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:45015/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733241442291/wal.1733241442409 took 43ms 2024-12-03T15:57:31,827 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:45015/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733241442291/wal.1733241442409 so closing down 2024-12-03T15:57:31,827 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-03T15:57:31,828 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-wal.1733241442409.temp 2024-12-03T15:57:31,872 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:45015/hbase/data/default/testReplayEditsAfterAbortingFlush/a0119fafb01eead8a6376bf521f59012/recovered.edits/0000000000000000004-wal.1733241442409.temp 2024-12-03T15:57:31,872 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-03T15:57:31,884 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44827 is added to blk_1073741904_1082 (size=3245) 2024-12-03T15:57:31,885 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40575 is added to blk_1073741904_1082 (size=3245) 2024-12-03T15:57:31,886 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46585 is added to blk_1073741904_1082 (size=3245) 2024-12-03T15:57:31,886 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:45015/hbase/data/default/testReplayEditsAfterAbortingFlush/a0119fafb01eead8a6376bf521f59012/recovered.edits/0000000000000000004-wal.1733241442409.temp (wrote 20 edits, skipped 0 edits in 0 ms) 2024-12-03T15:57:31,888 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:45015/hbase/data/default/testReplayEditsAfterAbortingFlush/a0119fafb01eead8a6376bf521f59012/recovered.edits/0000000000000000004-wal.1733241442409.temp to hdfs://localhost:45015/hbase/data/default/testReplayEditsAfterAbortingFlush/a0119fafb01eead8a6376bf521f59012/recovered.edits/0000000000000000026 2024-12-03T15:57:31,888 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 23 edits across 1 Regions in 66 ms; skipped=3; WAL=hdfs://localhost:45015/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733241442291/wal.1733241442409, size=4.1 K, length=4236, corrupted=false, cancelled=false 2024-12-03T15:57:31,889 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:45015/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733241442291/wal.1733241442409, journal: Splitting hdfs://localhost:45015/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733241442291/wal.1733241442409, size=4.1 K (4236bytes) at 1733241451778Finishing writing output for hdfs://localhost:45015/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733241442291/wal.1733241442409 so closing down at 1733241451827 (+49 ms)Creating recovered edits writer path=hdfs://localhost:45015/hbase/data/default/testReplayEditsAfterAbortingFlush/a0119fafb01eead8a6376bf521f59012/recovered.edits/0000000000000000004-wal.1733241442409.temp at 1733241451872 (+45 ms)3 split writer threads finished at 1733241451873 (+1 ms)Closed recovered edits writer path=hdfs://localhost:45015/hbase/data/default/testReplayEditsAfterAbortingFlush/a0119fafb01eead8a6376bf521f59012/recovered.edits/0000000000000000004-wal.1733241442409.temp (wrote 20 edits, skipped 0 edits in 0 ms) at 1733241451886 (+13 ms)Rename recovered edits hdfs://localhost:45015/hbase/data/default/testReplayEditsAfterAbortingFlush/a0119fafb01eead8a6376bf521f59012/recovered.edits/0000000000000000004-wal.1733241442409.temp to hdfs://localhost:45015/hbase/data/default/testReplayEditsAfterAbortingFlush/a0119fafb01eead8a6376bf521f59012/recovered.edits/0000000000000000026 at 1733241451888 (+2 ms)Processed 23 edits across 1 Regions in 66 ms; skipped=3; WAL=hdfs://localhost:45015/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733241442291/wal.1733241442409, size=4.1 K, length=4236, corrupted=false, cancelled=false at 1733241451888 2024-12-03T15:57:31,891 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:45015/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733241442291/wal.1733241442409 to hdfs://localhost:45015/hbase/oldWALs/wal.1733241442409 2024-12-03T15:57:31,892 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:45015/hbase/data/default/testReplayEditsAfterAbortingFlush/a0119fafb01eead8a6376bf521f59012/recovered.edits/0000000000000000026 2024-12-03T15:57:31,892 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-03T15:57:31,894 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:45015/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733241442291, archiveDir=hdfs://localhost:45015/hbase/oldWALs, maxLogs=32 2024-12-03T15:57:31,907 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733241442291/wal.1733241451894, exclude list is [], retry=0 2024-12-03T15:57:31,910 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44827,DS-e007f1cc-fc1b-4368-9716-e11e4fe2f4eb,DISK] 2024-12-03T15:57:31,911 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40575,DS-309fe9b7-1997-44cb-b0c9-1ce3c04a71de,DISK] 2024-12-03T15:57:31,911 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46585,DS-5c6771fd-75c7-4775-ad88-ae5bbe88de5c,DISK] 2024-12-03T15:57:31,953 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733241442291/wal.1733241451894 2024-12-03T15:57:31,957 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46731:46731),(127.0.0.1/127.0.0.1:44513:44513),(127.0.0.1/127.0.0.1:37355:37355)] 2024-12-03T15:57:31,958 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => a0119fafb01eead8a6376bf521f59012, NAME => 'testReplayEditsAfterAbortingFlush,,1733241442294.a0119fafb01eead8a6376bf521f59012.', STARTKEY => '', ENDKEY => ''} 2024-12-03T15:57:31,959 DEBUG [Time-limited test {}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterAbortingFlush a0119fafb01eead8a6376bf521f59012 2024-12-03T15:57:31,959 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterAbortingFlush,,1733241442294.a0119fafb01eead8a6376bf521f59012.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T15:57:31,959 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for a0119fafb01eead8a6376bf521f59012 2024-12-03T15:57:31,959 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for a0119fafb01eead8a6376bf521f59012 2024-12-03T15:57:31,961 INFO [StoreOpener-a0119fafb01eead8a6376bf521f59012-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region a0119fafb01eead8a6376bf521f59012 2024-12-03T15:57:31,963 INFO [StoreOpener-a0119fafb01eead8a6376bf521f59012-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a0119fafb01eead8a6376bf521f59012 columnFamilyName a 2024-12-03T15:57:31,963 DEBUG [StoreOpener-a0119fafb01eead8a6376bf521f59012-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T15:57:31,977 INFO [StoreOpener-a0119fafb01eead8a6376bf521f59012-1 {}] regionserver.HStore(327): Store=a0119fafb01eead8a6376bf521f59012/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T15:57:31,978 INFO [StoreOpener-a0119fafb01eead8a6376bf521f59012-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region a0119fafb01eead8a6376bf521f59012 2024-12-03T15:57:31,979 INFO [StoreOpener-a0119fafb01eead8a6376bf521f59012-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a0119fafb01eead8a6376bf521f59012 columnFamilyName b 2024-12-03T15:57:31,979 DEBUG [StoreOpener-a0119fafb01eead8a6376bf521f59012-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T15:57:31,980 INFO [StoreOpener-a0119fafb01eead8a6376bf521f59012-1 {}] regionserver.HStore(327): Store=a0119fafb01eead8a6376bf521f59012/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T15:57:31,981 INFO [StoreOpener-a0119fafb01eead8a6376bf521f59012-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region a0119fafb01eead8a6376bf521f59012 2024-12-03T15:57:31,982 INFO [StoreOpener-a0119fafb01eead8a6376bf521f59012-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a0119fafb01eead8a6376bf521f59012 columnFamilyName c 2024-12-03T15:57:31,982 DEBUG [StoreOpener-a0119fafb01eead8a6376bf521f59012-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T15:57:31,982 INFO [StoreOpener-a0119fafb01eead8a6376bf521f59012-1 {}] regionserver.HStore(327): Store=a0119fafb01eead8a6376bf521f59012/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T15:57:31,983 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for a0119fafb01eead8a6376bf521f59012 2024-12-03T15:57:31,983 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45015/hbase/data/default/testReplayEditsAfterAbortingFlush/a0119fafb01eead8a6376bf521f59012 2024-12-03T15:57:31,985 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:45015/hbase/data/default/testReplayEditsAfterAbortingFlush/a0119fafb01eead8a6376bf521f59012 2024-12-03T15:57:31,986 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:45015/hbase/data/default/testReplayEditsAfterAbortingFlush/a0119fafb01eead8a6376bf521f59012/recovered.edits/0000000000000000026 2024-12-03T15:57:32,026 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-03T15:57:32,028 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 20, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=26, path=hdfs://localhost:45015/hbase/data/default/testReplayEditsAfterAbortingFlush/a0119fafb01eead8a6376bf521f59012/recovered.edits/0000000000000000026 2024-12-03T15:57:32,028 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing a0119fafb01eead8a6376bf521f59012 3/3 column families, dataSize=1.16 KB heapSize=3.41 KB 2024-12-03T15:57:32,046 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45015/hbase/data/default/testReplayEditsAfterAbortingFlush/a0119fafb01eead8a6376bf521f59012/.tmp/a/8ccc769672aa4495bdaa8eae4a1acfad is 64, key is testReplayEditsAfterAbortingFlush12/a:q/1733241451634/Put/seqid=0 2024-12-03T15:57:32,055 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46585 is added to blk_1073741906_1084 (size=5523) 2024-12-03T15:57:32,055 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40575 is added to blk_1073741906_1084 (size=5523) 2024-12-03T15:57:32,056 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=416 B at sequenceid=26 (bloomFilter=true), to=hdfs://localhost:45015/hbase/data/default/testReplayEditsAfterAbortingFlush/a0119fafb01eead8a6376bf521f59012/.tmp/a/8ccc769672aa4495bdaa8eae4a1acfad 2024-12-03T15:57:32,059 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44827 is added to blk_1073741906_1084 (size=5523) 2024-12-03T15:57:32,087 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45015/hbase/data/default/testReplayEditsAfterAbortingFlush/a0119fafb01eead8a6376bf521f59012/.tmp/b/9f6ef6a5385545cfb9de57b95f69b679 is 64, key is testReplayEditsAfterAbortingFlush10/b:q/1733241451628/Put/seqid=0 2024-12-03T15:57:32,108 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46585 is added to blk_1073741907_1085 (size=5524) 2024-12-03T15:57:32,108 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40575 is added to blk_1073741907_1085 (size=5524) 2024-12-03T15:57:32,109 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44827 is added to blk_1073741907_1085 (size=5524) 2024-12-03T15:57:32,110 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=417 B at sequenceid=26 (bloomFilter=true), to=hdfs://localhost:45015/hbase/data/default/testReplayEditsAfterAbortingFlush/a0119fafb01eead8a6376bf521f59012/.tmp/b/9f6ef6a5385545cfb9de57b95f69b679 2024-12-03T15:57:32,137 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45015/hbase/data/default/testReplayEditsAfterAbortingFlush/a0119fafb01eead8a6376bf521f59012/.tmp/c/d0a6c59ae9c84c7c8ac4e35cbc010611 is 64, key is testReplayEditsAfterAbortingFlush11/c:q/1733241451631/Put/seqid=0 2024-12-03T15:57:32,148 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44827 is added to blk_1073741908_1086 (size=5457) 2024-12-03T15:57:32,150 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=357 B at sequenceid=26 (bloomFilter=true), to=hdfs://localhost:45015/hbase/data/default/testReplayEditsAfterAbortingFlush/a0119fafb01eead8a6376bf521f59012/.tmp/c/d0a6c59ae9c84c7c8ac4e35cbc010611 2024-12-03T15:57:32,150 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46585 is added to blk_1073741908_1086 (size=5457) 2024-12-03T15:57:32,150 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40575 is added to blk_1073741908_1086 (size=5457) 2024-12-03T15:57:32,158 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45015/hbase/data/default/testReplayEditsAfterAbortingFlush/a0119fafb01eead8a6376bf521f59012/.tmp/a/8ccc769672aa4495bdaa8eae4a1acfad as hdfs://localhost:45015/hbase/data/default/testReplayEditsAfterAbortingFlush/a0119fafb01eead8a6376bf521f59012/a/8ccc769672aa4495bdaa8eae4a1acfad 2024-12-03T15:57:32,165 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45015/hbase/data/default/testReplayEditsAfterAbortingFlush/a0119fafb01eead8a6376bf521f59012/a/8ccc769672aa4495bdaa8eae4a1acfad, entries=7, sequenceid=26, filesize=5.4 K 2024-12-03T15:57:32,166 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45015/hbase/data/default/testReplayEditsAfterAbortingFlush/a0119fafb01eead8a6376bf521f59012/.tmp/b/9f6ef6a5385545cfb9de57b95f69b679 as hdfs://localhost:45015/hbase/data/default/testReplayEditsAfterAbortingFlush/a0119fafb01eead8a6376bf521f59012/b/9f6ef6a5385545cfb9de57b95f69b679 2024-12-03T15:57:32,173 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45015/hbase/data/default/testReplayEditsAfterAbortingFlush/a0119fafb01eead8a6376bf521f59012/b/9f6ef6a5385545cfb9de57b95f69b679, entries=7, sequenceid=26, filesize=5.4 K 2024-12-03T15:57:32,175 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45015/hbase/data/default/testReplayEditsAfterAbortingFlush/a0119fafb01eead8a6376bf521f59012/.tmp/c/d0a6c59ae9c84c7c8ac4e35cbc010611 as hdfs://localhost:45015/hbase/data/default/testReplayEditsAfterAbortingFlush/a0119fafb01eead8a6376bf521f59012/c/d0a6c59ae9c84c7c8ac4e35cbc010611 2024-12-03T15:57:32,181 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45015/hbase/data/default/testReplayEditsAfterAbortingFlush/a0119fafb01eead8a6376bf521f59012/c/d0a6c59ae9c84c7c8ac4e35cbc010611, entries=6, sequenceid=26, filesize=5.3 K 2024-12-03T15:57:32,181 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~1.16 KB/1190, heapSize ~3.36 KB/3440, currentSize=0 B/0 for a0119fafb01eead8a6376bf521f59012 in 153ms, sequenceid=26, compaction requested=false; wal=null 2024-12-03T15:57:32,182 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:45015/hbase/data/default/testReplayEditsAfterAbortingFlush/a0119fafb01eead8a6376bf521f59012/recovered.edits/0000000000000000026 2024-12-03T15:57:32,183 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for a0119fafb01eead8a6376bf521f59012 2024-12-03T15:57:32,183 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for a0119fafb01eead8a6376bf521f59012 2024-12-03T15:57:32,184 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterAbortingFlush descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-03T15:57:32,186 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for a0119fafb01eead8a6376bf521f59012 2024-12-03T15:57:32,188 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45015/hbase/data/default/testReplayEditsAfterAbortingFlush/a0119fafb01eead8a6376bf521f59012/recovered.edits/26.seqid, newMaxSeqId=26, maxSeqId=1 2024-12-03T15:57:32,189 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened a0119fafb01eead8a6376bf521f59012; next sequenceid=27; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72261712, jitterRate=0.07678341865539551}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-03T15:57:32,190 DEBUG [Time-limited test {}] regionserver.HRegion(1122): Running coprocessor post-open hooks for a0119fafb01eead8a6376bf521f59012 2024-12-03T15:57:32,190 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for a0119fafb01eead8a6376bf521f59012: Running coprocessor pre-open hook at 1733241451959Writing region info on filesystem at 1733241451959Initializing all the Stores at 1733241451961 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733241451961Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733241451961Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733241451961Obtaining lock to block concurrent updates at 1733241452028 (+67 ms)Preparing flush snapshotting stores in a0119fafb01eead8a6376bf521f59012 at 1733241452028Finished memstore snapshotting testReplayEditsAfterAbortingFlush,,1733241442294.a0119fafb01eead8a6376bf521f59012., syncing WAL and waiting on mvcc, flushsize=dataSize=1190, getHeapSize=3440, getOffHeapSize=0, getCellsCount=20 at 1733241452029 (+1 ms)Flushing stores of testReplayEditsAfterAbortingFlush,,1733241442294.a0119fafb01eead8a6376bf521f59012. at 1733241452029Flushing a0119fafb01eead8a6376bf521f59012/a: creating writer at 1733241452029Flushing a0119fafb01eead8a6376bf521f59012/a: appending metadata at 1733241452045 (+16 ms)Flushing a0119fafb01eead8a6376bf521f59012/a: closing flushed file at 1733241452046 (+1 ms)Flushing a0119fafb01eead8a6376bf521f59012/b: creating writer at 1733241452070 (+24 ms)Flushing a0119fafb01eead8a6376bf521f59012/b: appending metadata at 1733241452087 (+17 ms)Flushing a0119fafb01eead8a6376bf521f59012/b: closing flushed file at 1733241452087Flushing a0119fafb01eead8a6376bf521f59012/c: creating writer at 1733241452116 (+29 ms)Flushing a0119fafb01eead8a6376bf521f59012/c: appending metadata at 1733241452137 (+21 ms)Flushing a0119fafb01eead8a6376bf521f59012/c: closing flushed file at 1733241452137Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@326a40c8: reopening flushed file at 1733241452157 (+20 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2b95ce9f: reopening flushed file at 1733241452165 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@10988281: reopening flushed file at 1733241452174 (+9 ms)Finished flush of dataSize ~1.16 KB/1190, heapSize ~3.36 KB/3440, currentSize=0 B/0 for a0119fafb01eead8a6376bf521f59012 in 153ms, sequenceid=26, compaction requested=false; wal=null at 1733241452181 (+7 ms)Cleaning up temporary data from old regions at 1733241452183 (+2 ms)Running coprocessor post-open hooks at 1733241452190 (+7 ms)Region opened successfully at 1733241452190 2024-12-03T15:57:32,217 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsAfterAbortingFlush Thread=419 (was 418) Potentially hanging thread: PacketResponder: BP-1122350750-172.17.0.3-1733241413817:blk_1073741905_1083, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-365845159_22 at /127.0.0.1:35224 [Waiting for operation #5] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-365845159_22 at /127.0.0.1:41824 [Receiving block BP-1122350750-172.17.0.3-1733241413817:blk_1073741905_1083] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1122350750-172.17.0.3-1733241413817:blk_1073741905_1083, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-365845159_22 at /127.0.0.1:35188 [Receiving block BP-1122350750-172.17.0.3-1733241413817:blk_1073741905_1083] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-365845159_22 at /127.0.0.1:60704 [Receiving block BP-1122350750-172.17.0.3-1733241413817:blk_1073741905_1083] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-22-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-22-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-365845159_22 at /127.0.0.1:41800 [Waiting for operation #9] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1122350750-172.17.0.3-1733241413817:blk_1073741905_1083, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-22-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-365845159_22 at /127.0.0.1:60712 [Waiting for operation #11] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1186 (was 1119) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=776 (was 827), ProcessCount=11 (was 11), AvailableMemoryMB=4795 (was 4939) 2024-12-03T15:57:32,218 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1186 is superior to 1024 2024-12-03T15:57:32,237 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testDatalossWhenInputError Thread=419, OpenFileDescriptor=1186, MaxFileDescriptor=1048576, SystemLoadAverage=776, ProcessCount=11, AvailableMemoryMB=4790 2024-12-03T15:57:32,237 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1186 is superior to 1024 2024-12-03T15:57:32,254 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-03T15:57:32,256 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-03T15:57:32,256 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-03T15:57:32,259 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-45894462, suffix=, logDir=hdfs://localhost:45015/hbase/WALs/hregion-45894462, archiveDir=hdfs://localhost:45015/hbase/oldWALs, maxLogs=32 2024-12-03T15:57:32,273 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-45894462/hregion-45894462.1733241452259, exclude list is [], retry=0 2024-12-03T15:57:32,276 DEBUG [AsyncFSWAL-24-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40575,DS-309fe9b7-1997-44cb-b0c9-1ce3c04a71de,DISK] 2024-12-03T15:57:32,277 DEBUG [AsyncFSWAL-24-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44827,DS-e007f1cc-fc1b-4368-9716-e11e4fe2f4eb,DISK] 2024-12-03T15:57:32,277 DEBUG [AsyncFSWAL-24-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46585,DS-5c6771fd-75c7-4775-ad88-ae5bbe88de5c,DISK] 2024-12-03T15:57:32,309 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-45894462/hregion-45894462.1733241452259 2024-12-03T15:57:32,310 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44513:44513),(127.0.0.1/127.0.0.1:46731:46731),(127.0.0.1/127.0.0.1:37355:37355)] 2024-12-03T15:57:32,310 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 9783daaf864701d878cef89331db5cf5, NAME => 'testDatalossWhenInputError,,1733241452254.9783daaf864701d878cef89331db5cf5.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testDatalossWhenInputError', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45015/hbase 2024-12-03T15:57:32,319 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44827 is added to blk_1073741910_1088 (size=61) 2024-12-03T15:57:32,319 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46585 is added to blk_1073741910_1088 (size=61) 2024-12-03T15:57:32,320 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40575 is added to blk_1073741910_1088 (size=61) 2024-12-03T15:57:32,320 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1733241452254.9783daaf864701d878cef89331db5cf5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T15:57:32,322 INFO [StoreOpener-9783daaf864701d878cef89331db5cf5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 9783daaf864701d878cef89331db5cf5 2024-12-03T15:57:32,323 INFO [StoreOpener-9783daaf864701d878cef89331db5cf5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9783daaf864701d878cef89331db5cf5 columnFamilyName a 2024-12-03T15:57:32,323 DEBUG [StoreOpener-9783daaf864701d878cef89331db5cf5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T15:57:32,323 INFO [StoreOpener-9783daaf864701d878cef89331db5cf5-1 {}] regionserver.HStore(327): Store=9783daaf864701d878cef89331db5cf5/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T15:57:32,324 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 9783daaf864701d878cef89331db5cf5 2024-12-03T15:57:32,324 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45015/hbase/data/default/testDatalossWhenInputError/9783daaf864701d878cef89331db5cf5 2024-12-03T15:57:32,325 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45015/hbase/data/default/testDatalossWhenInputError/9783daaf864701d878cef89331db5cf5 2024-12-03T15:57:32,325 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 9783daaf864701d878cef89331db5cf5 2024-12-03T15:57:32,325 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 9783daaf864701d878cef89331db5cf5 2024-12-03T15:57:32,327 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 9783daaf864701d878cef89331db5cf5 2024-12-03T15:57:32,328 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45015/hbase/data/default/testDatalossWhenInputError/9783daaf864701d878cef89331db5cf5/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T15:57:32,329 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 9783daaf864701d878cef89331db5cf5; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73052080, jitterRate=0.08856081962585449}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-03T15:57:32,329 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 9783daaf864701d878cef89331db5cf5: Writing region info on filesystem at 1733241452320Initializing all the Stores at 1733241452321 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733241452321Cleaning up temporary data from old regions at 1733241452325 (+4 ms)Region opened successfully at 1733241452329 (+4 ms) 2024-12-03T15:57:32,329 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 9783daaf864701d878cef89331db5cf5, disabling compactions & flushes 2024-12-03T15:57:32,330 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testDatalossWhenInputError,,1733241452254.9783daaf864701d878cef89331db5cf5. 2024-12-03T15:57:32,330 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testDatalossWhenInputError,,1733241452254.9783daaf864701d878cef89331db5cf5. 2024-12-03T15:57:32,330 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testDatalossWhenInputError,,1733241452254.9783daaf864701d878cef89331db5cf5. after waiting 0 ms 2024-12-03T15:57:32,330 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testDatalossWhenInputError,,1733241452254.9783daaf864701d878cef89331db5cf5. 2024-12-03T15:57:32,330 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testDatalossWhenInputError,,1733241452254.9783daaf864701d878cef89331db5cf5. 2024-12-03T15:57:32,330 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 9783daaf864701d878cef89331db5cf5: Waiting for close lock at 1733241452329Disabling compacts and flushes for region at 1733241452329Disabling writes for close at 1733241452330 (+1 ms)Writing region close event to WAL at 1733241452330Closed at 1733241452330 2024-12-03T15:57:32,332 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40575 is added to blk_1073741909_1087 (size=175) 2024-12-03T15:57:32,333 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46585 is added to blk_1073741909_1087 (size=175) 2024-12-03T15:57:32,333 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44827 is added to blk_1073741909_1087 (size=175) 2024-12-03T15:57:32,335 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-03T15:57:32,335 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-45894462:(num 1733241452259) 2024-12-03T15:57:32,335 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-03T15:57:32,337 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:45015/hbase/WALs/testdatalosswheninputerror-manual,16010,1733241452253, archiveDir=hdfs://localhost:45015/hbase/oldWALs, maxLogs=32 2024-12-03T15:57:32,352 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testdatalosswheninputerror-manual,16010,1733241452253/wal.1733241452337, exclude list is [], retry=0 2024-12-03T15:57:32,355 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40575,DS-309fe9b7-1997-44cb-b0c9-1ce3c04a71de,DISK] 2024-12-03T15:57:32,355 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46585,DS-5c6771fd-75c7-4775-ad88-ae5bbe88de5c,DISK] 2024-12-03T15:57:32,355 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44827,DS-e007f1cc-fc1b-4368-9716-e11e4fe2f4eb,DISK] 2024-12-03T15:57:32,410 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testdatalosswheninputerror-manual,16010,1733241452253/wal.1733241452337 2024-12-03T15:57:32,411 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44513:44513),(127.0.0.1/127.0.0.1:37355:37355),(127.0.0.1/127.0.0.1:46731:46731)] 2024-12-03T15:57:32,411 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 9783daaf864701d878cef89331db5cf5, NAME => 'testDatalossWhenInputError,,1733241452254.9783daaf864701d878cef89331db5cf5.', STARTKEY => '', ENDKEY => ''} 2024-12-03T15:57:32,411 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1733241452254.9783daaf864701d878cef89331db5cf5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T15:57:32,411 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 9783daaf864701d878cef89331db5cf5 2024-12-03T15:57:32,411 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 9783daaf864701d878cef89331db5cf5 2024-12-03T15:57:32,414 INFO [StoreOpener-9783daaf864701d878cef89331db5cf5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 9783daaf864701d878cef89331db5cf5 2024-12-03T15:57:32,415 INFO [StoreOpener-9783daaf864701d878cef89331db5cf5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9783daaf864701d878cef89331db5cf5 columnFamilyName a 2024-12-03T15:57:32,415 DEBUG [StoreOpener-9783daaf864701d878cef89331db5cf5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T15:57:32,416 INFO [StoreOpener-9783daaf864701d878cef89331db5cf5-1 {}] regionserver.HStore(327): Store=9783daaf864701d878cef89331db5cf5/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T15:57:32,416 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 9783daaf864701d878cef89331db5cf5 2024-12-03T15:57:32,417 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45015/hbase/data/default/testDatalossWhenInputError/9783daaf864701d878cef89331db5cf5 2024-12-03T15:57:32,418 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45015/hbase/data/default/testDatalossWhenInputError/9783daaf864701d878cef89331db5cf5 2024-12-03T15:57:32,418 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 9783daaf864701d878cef89331db5cf5 2024-12-03T15:57:32,418 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 9783daaf864701d878cef89331db5cf5 2024-12-03T15:57:32,420 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 9783daaf864701d878cef89331db5cf5 2024-12-03T15:57:32,421 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 9783daaf864701d878cef89331db5cf5; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74121884, jitterRate=0.10450214147567749}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-03T15:57:32,422 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 9783daaf864701d878cef89331db5cf5: Writing region info on filesystem at 1733241452411Initializing all the Stores at 1733241452412 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733241452412Cleaning up temporary data from old regions at 1733241452418 (+6 ms)Region opened successfully at 1733241452422 (+4 ms) 2024-12-03T15:57:32,433 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 9783daaf864701d878cef89331db5cf5, disabling compactions & flushes 2024-12-03T15:57:32,433 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testDatalossWhenInputError,,1733241452254.9783daaf864701d878cef89331db5cf5. 2024-12-03T15:57:32,433 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testDatalossWhenInputError,,1733241452254.9783daaf864701d878cef89331db5cf5. 2024-12-03T15:57:32,433 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testDatalossWhenInputError,,1733241452254.9783daaf864701d878cef89331db5cf5. after waiting 0 ms 2024-12-03T15:57:32,433 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testDatalossWhenInputError,,1733241452254.9783daaf864701d878cef89331db5cf5. 2024-12-03T15:57:32,433 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 750 in region testDatalossWhenInputError,,1733241452254.9783daaf864701d878cef89331db5cf5. 2024-12-03T15:57:32,433 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testDatalossWhenInputError,,1733241452254.9783daaf864701d878cef89331db5cf5. 2024-12-03T15:57:32,433 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 9783daaf864701d878cef89331db5cf5: Waiting for close lock at 1733241452433Disabling compacts and flushes for region at 1733241452433Disabling writes for close at 1733241452433Writing region close event to WAL at 1733241452433Closed at 1733241452433 2024-12-03T15:57:32,436 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40575 is added to blk_1073741911_1089 (size=1795) 2024-12-03T15:57:32,436 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44827 is added to blk_1073741911_1089 (size=1795) 2024-12-03T15:57:32,436 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46585 is added to blk_1073741911_1089 (size=1795) 2024-12-03T15:57:32,453 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:45015/hbase/WALs/testdatalosswheninputerror-manual,16010,1733241452253/wal.1733241452337, size=1.8 K (1795bytes) 2024-12-03T15:57:32,453 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:45015/hbase/WALs/testdatalosswheninputerror-manual,16010,1733241452253/wal.1733241452337 2024-12-03T15:57:32,453 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:45015/hbase/WALs/testdatalosswheninputerror-manual,16010,1733241452253/wal.1733241452337 after 0ms 2024-12-03T15:57:32,482 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-03T15:57:32,483 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:45015/hbase/WALs/testdatalosswheninputerror-manual,16010,1733241452253/wal.1733241452337 took 29ms 2024-12-03T15:57:32,485 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:45015/hbase/WALs/testdatalosswheninputerror-manual,16010,1733241452253/wal.1733241452337 so closing down 2024-12-03T15:57:32,485 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-03T15:57:32,486 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1733241452337.temp 2024-12-03T15:57:32,515 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:45015/hbase/data/default/testDatalossWhenInputError/9783daaf864701d878cef89331db5cf5/recovered.edits/0000000000000000003-wal.1733241452337.temp 2024-12-03T15:57:32,515 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-03T15:57:32,522 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40575 is added to blk_1073741912_1090 (size=1795) 2024-12-03T15:57:32,523 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44827 is added to blk_1073741912_1090 (size=1795) 2024-12-03T15:57:32,524 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46585 is added to blk_1073741912_1090 (size=1795) 2024-12-03T15:57:32,526 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:45015/hbase/data/default/testDatalossWhenInputError/9783daaf864701d878cef89331db5cf5/recovered.edits/0000000000000000003-wal.1733241452337.temp (wrote 10 edits, skipped 0 edits in 0 ms) 2024-12-03T15:57:32,530 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:45015/hbase/data/default/testDatalossWhenInputError/9783daaf864701d878cef89331db5cf5/recovered.edits/0000000000000000003-wal.1733241452337.temp to hdfs://localhost:45015/hbase/data/default/testDatalossWhenInputError/9783daaf864701d878cef89331db5cf5/recovered.edits/0000000000000000012 2024-12-03T15:57:32,530 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 10 edits across 1 Regions in 47 ms; skipped=0; WAL=hdfs://localhost:45015/hbase/WALs/testdatalosswheninputerror-manual,16010,1733241452253/wal.1733241452337, size=1.8 K, length=1795, corrupted=false, cancelled=false 2024-12-03T15:57:32,530 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:45015/hbase/WALs/testdatalosswheninputerror-manual,16010,1733241452253/wal.1733241452337, journal: Splitting hdfs://localhost:45015/hbase/WALs/testdatalosswheninputerror-manual,16010,1733241452253/wal.1733241452337, size=1.8 K (1795bytes) at 1733241452453Finishing writing output for hdfs://localhost:45015/hbase/WALs/testdatalosswheninputerror-manual,16010,1733241452253/wal.1733241452337 so closing down at 1733241452485 (+32 ms)Creating recovered edits writer path=hdfs://localhost:45015/hbase/data/default/testDatalossWhenInputError/9783daaf864701d878cef89331db5cf5/recovered.edits/0000000000000000003-wal.1733241452337.temp at 1733241452515 (+30 ms)3 split writer threads finished at 1733241452515Closed recovered edits writer path=hdfs://localhost:45015/hbase/data/default/testDatalossWhenInputError/9783daaf864701d878cef89331db5cf5/recovered.edits/0000000000000000003-wal.1733241452337.temp (wrote 10 edits, skipped 0 edits in 0 ms) at 1733241452526 (+11 ms)Rename recovered edits hdfs://localhost:45015/hbase/data/default/testDatalossWhenInputError/9783daaf864701d878cef89331db5cf5/recovered.edits/0000000000000000003-wal.1733241452337.temp to hdfs://localhost:45015/hbase/data/default/testDatalossWhenInputError/9783daaf864701d878cef89331db5cf5/recovered.edits/0000000000000000012 at 1733241452530 (+4 ms)Processed 10 edits across 1 Regions in 47 ms; skipped=0; WAL=hdfs://localhost:45015/hbase/WALs/testdatalosswheninputerror-manual,16010,1733241452253/wal.1733241452337, size=1.8 K, length=1795, corrupted=false, cancelled=false at 1733241452530 2024-12-03T15:57:32,532 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:45015/hbase/WALs/testdatalosswheninputerror-manual,16010,1733241452253/wal.1733241452337 to hdfs://localhost:45015/hbase/oldWALs/wal.1733241452337 2024-12-03T15:57:32,533 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:45015/hbase/data/default/testDatalossWhenInputError/9783daaf864701d878cef89331db5cf5/recovered.edits/0000000000000000012 2024-12-03T15:57:32,567 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-03T15:57:32,968 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-03T15:57:32,971 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:45015/hbase/WALs/testdatalosswheninputerror-manual,16010,1733241452253, archiveDir=hdfs://localhost:45015/hbase/oldWALs, maxLogs=32 2024-12-03T15:57:32,984 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testdatalosswheninputerror-manual,16010,1733241452253/wal.1733241452971, exclude list is [], retry=0 2024-12-03T15:57:32,987 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44827,DS-e007f1cc-fc1b-4368-9716-e11e4fe2f4eb,DISK] 2024-12-03T15:57:32,988 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40575,DS-309fe9b7-1997-44cb-b0c9-1ce3c04a71de,DISK] 2024-12-03T15:57:32,988 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46585,DS-5c6771fd-75c7-4775-ad88-ae5bbe88de5c,DISK] 2024-12-03T15:57:33,016 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testdatalosswheninputerror-manual,16010,1733241452253/wal.1733241452971 2024-12-03T15:57:33,016 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46731:46731),(127.0.0.1/127.0.0.1:44513:44513),(127.0.0.1/127.0.0.1:37355:37355)] 2024-12-03T15:57:33,016 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 9783daaf864701d878cef89331db5cf5, NAME => 'testDatalossWhenInputError,,1733241452254.9783daaf864701d878cef89331db5cf5.', STARTKEY => '', ENDKEY => ''} 2024-12-03T15:57:33,017 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1733241452254.9783daaf864701d878cef89331db5cf5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T15:57:33,017 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 9783daaf864701d878cef89331db5cf5 2024-12-03T15:57:33,017 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 9783daaf864701d878cef89331db5cf5 2024-12-03T15:57:33,020 INFO [StoreOpener-9783daaf864701d878cef89331db5cf5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 9783daaf864701d878cef89331db5cf5 2024-12-03T15:57:33,022 INFO [StoreOpener-9783daaf864701d878cef89331db5cf5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9783daaf864701d878cef89331db5cf5 columnFamilyName a 2024-12-03T15:57:33,022 DEBUG [StoreOpener-9783daaf864701d878cef89331db5cf5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T15:57:33,023 INFO [StoreOpener-9783daaf864701d878cef89331db5cf5-1 {}] regionserver.HStore(327): Store=9783daaf864701d878cef89331db5cf5/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T15:57:33,023 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 9783daaf864701d878cef89331db5cf5 2024-12-03T15:57:33,023 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45015/hbase/data/default/testDatalossWhenInputError/9783daaf864701d878cef89331db5cf5 2024-12-03T15:57:33,025 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:45015/hbase/data/default/testDatalossWhenInputError/9783daaf864701d878cef89331db5cf5 2024-12-03T15:57:33,025 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:45015/hbase/data/default/testDatalossWhenInputError/9783daaf864701d878cef89331db5cf5/recovered.edits/0000000000000000012 2024-12-03T15:57:33,055 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-03T15:57:33,057 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 10, skipped 0, firstSequenceIdInLog=3, maxSequenceIdInLog=12, path=hdfs://localhost:45015/hbase/data/default/testDatalossWhenInputError/9783daaf864701d878cef89331db5cf5/recovered.edits/0000000000000000012 2024-12-03T15:57:33,057 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 9783daaf864701d878cef89331db5cf5 1/1 column families, dataSize=750 B heapSize=1.73 KB 2024-12-03T15:57:33,094 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45015/hbase/data/default/testDatalossWhenInputError/9783daaf864701d878cef89331db5cf5/.tmp/a/dcbe86e87c3340b2810faf96ea971cce is 79, key is testDatalossWhenInputError/a:x0/1733241452422/Put/seqid=0 2024-12-03T15:57:33,113 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40575 is added to blk_1073741914_1092 (size=5808) 2024-12-03T15:57:33,114 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46585 is added to blk_1073741914_1092 (size=5808) 2024-12-03T15:57:33,115 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=750 B at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:45015/hbase/data/default/testDatalossWhenInputError/9783daaf864701d878cef89331db5cf5/.tmp/a/dcbe86e87c3340b2810faf96ea971cce 2024-12-03T15:57:33,118 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44827 is added to blk_1073741914_1092 (size=5808) 2024-12-03T15:57:33,135 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45015/hbase/data/default/testDatalossWhenInputError/9783daaf864701d878cef89331db5cf5/.tmp/a/dcbe86e87c3340b2810faf96ea971cce as hdfs://localhost:45015/hbase/data/default/testDatalossWhenInputError/9783daaf864701d878cef89331db5cf5/a/dcbe86e87c3340b2810faf96ea971cce 2024-12-03T15:57:33,154 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45015/hbase/data/default/testDatalossWhenInputError/9783daaf864701d878cef89331db5cf5/a/dcbe86e87c3340b2810faf96ea971cce, entries=10, sequenceid=12, filesize=5.7 K 2024-12-03T15:57:33,154 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~750 B/750, heapSize ~1.72 KB/1760, currentSize=0 B/0 for 9783daaf864701d878cef89331db5cf5 in 97ms, sequenceid=12, compaction requested=false; wal=null 2024-12-03T15:57:33,155 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:45015/hbase/data/default/testDatalossWhenInputError/9783daaf864701d878cef89331db5cf5/recovered.edits/0000000000000000012 2024-12-03T15:57:33,156 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 9783daaf864701d878cef89331db5cf5 2024-12-03T15:57:33,156 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 9783daaf864701d878cef89331db5cf5 2024-12-03T15:57:33,159 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 9783daaf864701d878cef89331db5cf5 2024-12-03T15:57:33,170 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45015/hbase/data/default/testDatalossWhenInputError/9783daaf864701d878cef89331db5cf5/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=1 2024-12-03T15:57:33,172 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 9783daaf864701d878cef89331db5cf5; next sequenceid=13; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71189654, jitterRate=0.06080850958824158}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-03T15:57:33,173 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 9783daaf864701d878cef89331db5cf5: Writing region info on filesystem at 1733241453017Initializing all the Stores at 1733241453020 (+3 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733241453020Obtaining lock to block concurrent updates at 1733241453057 (+37 ms)Preparing flush snapshotting stores in 9783daaf864701d878cef89331db5cf5 at 1733241453057Finished memstore snapshotting testDatalossWhenInputError,,1733241452254.9783daaf864701d878cef89331db5cf5., syncing WAL and waiting on mvcc, flushsize=dataSize=750, getHeapSize=1760, getOffHeapSize=0, getCellsCount=10 at 1733241453057Flushing stores of testDatalossWhenInputError,,1733241452254.9783daaf864701d878cef89331db5cf5. at 1733241453057Flushing 9783daaf864701d878cef89331db5cf5/a: creating writer at 1733241453057Flushing 9783daaf864701d878cef89331db5cf5/a: appending metadata at 1733241453093 (+36 ms)Flushing 9783daaf864701d878cef89331db5cf5/a: closing flushed file at 1733241453093Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@10ec7f2f: reopening flushed file at 1733241453131 (+38 ms)Finished flush of dataSize ~750 B/750, heapSize ~1.72 KB/1760, currentSize=0 B/0 for 9783daaf864701d878cef89331db5cf5 in 97ms, sequenceid=12, compaction requested=false; wal=null at 1733241453154 (+23 ms)Cleaning up temporary data from old regions at 1733241453156 (+2 ms)Region opened successfully at 1733241453173 (+17 ms) 2024-12-03T15:57:33,177 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 9783daaf864701d878cef89331db5cf5, NAME => 'testDatalossWhenInputError,,1733241452254.9783daaf864701d878cef89331db5cf5.', STARTKEY => '', ENDKEY => ''} 2024-12-03T15:57:33,177 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1733241452254.9783daaf864701d878cef89331db5cf5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T15:57:33,178 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 9783daaf864701d878cef89331db5cf5 2024-12-03T15:57:33,178 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 9783daaf864701d878cef89331db5cf5 2024-12-03T15:57:33,185 INFO [StoreOpener-9783daaf864701d878cef89331db5cf5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 9783daaf864701d878cef89331db5cf5 2024-12-03T15:57:33,187 INFO [StoreOpener-9783daaf864701d878cef89331db5cf5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9783daaf864701d878cef89331db5cf5 columnFamilyName a 2024-12-03T15:57:33,187 DEBUG [StoreOpener-9783daaf864701d878cef89331db5cf5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T15:57:33,198 DEBUG [StoreOpener-9783daaf864701d878cef89331db5cf5-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:45015/hbase/data/default/testDatalossWhenInputError/9783daaf864701d878cef89331db5cf5/a/dcbe86e87c3340b2810faf96ea971cce 2024-12-03T15:57:33,198 INFO [StoreOpener-9783daaf864701d878cef89331db5cf5-1 {}] regionserver.HStore(327): Store=9783daaf864701d878cef89331db5cf5/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T15:57:33,199 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 9783daaf864701d878cef89331db5cf5 2024-12-03T15:57:33,200 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45015/hbase/data/default/testDatalossWhenInputError/9783daaf864701d878cef89331db5cf5 2024-12-03T15:57:33,201 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45015/hbase/data/default/testDatalossWhenInputError/9783daaf864701d878cef89331db5cf5 2024-12-03T15:57:33,202 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 9783daaf864701d878cef89331db5cf5 2024-12-03T15:57:33,202 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 9783daaf864701d878cef89331db5cf5 2024-12-03T15:57:33,206 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 9783daaf864701d878cef89331db5cf5 2024-12-03T15:57:33,209 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45015/hbase/data/default/testDatalossWhenInputError/9783daaf864701d878cef89331db5cf5/recovered.edits/13.seqid, newMaxSeqId=13, maxSeqId=12 2024-12-03T15:57:33,213 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 9783daaf864701d878cef89331db5cf5; next sequenceid=14; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73630504, jitterRate=0.09718000888824463}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-03T15:57:33,213 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 9783daaf864701d878cef89331db5cf5: Writing region info on filesystem at 1733241453178Initializing all the Stores at 1733241453179 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733241453179Cleaning up temporary data from old regions at 1733241453202 (+23 ms)Region opened successfully at 1733241453213 (+11 ms) 2024-12-03T15:57:33,240 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testDatalossWhenInputError Thread=429 (was 419) Potentially hanging thread: PacketResponder: BP-1122350750-172.17.0.3-1733241413817:blk_1073741913_1091, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-365845159_22 at /127.0.0.1:60792 [Receiving block BP-1122350750-172.17.0.3-1733241413817:blk_1073741913_1091] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-365845159_22 at /127.0.0.1:35224 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-365845159_22 at /127.0.0.1:41888 [Receiving block BP-1122350750-172.17.0.3-1733241413817:blk_1073741913_1091] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-24-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-365845159_22 at /127.0.0.1:41800 [Waiting for operation #14] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1122350750-172.17.0.3-1733241413817:blk_1073741913_1091, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1122350750-172.17.0.3-1733241413817:blk_1073741913_1091, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-24-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-24-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-365845159_22 at /127.0.0.1:35274 [Receiving block BP-1122350750-172.17.0.3-1733241413817:blk_1073741913_1091] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-365845159_22 at /127.0.0.1:60712 [Waiting for operation #21] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1268 (was 1186) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=776 (was 776), ProcessCount=11 (was 11), AvailableMemoryMB=4855 (was 4790) - AvailableMemoryMB LEAK? - 2024-12-03T15:57:33,240 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1268 is superior to 1024 2024-12-03T15:57:33,254 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testCompactedBulkLoadedFiles Thread=429, OpenFileDescriptor=1268, MaxFileDescriptor=1048576, SystemLoadAverage=776, ProcessCount=11, AvailableMemoryMB=4854 2024-12-03T15:57:33,254 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1268 is superior to 1024 2024-12-03T15:57:33,270 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-03T15:57:33,271 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-03T15:57:33,272 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-03T15:57:33,275 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-82468117, suffix=, logDir=hdfs://localhost:45015/hbase/WALs/hregion-82468117, archiveDir=hdfs://localhost:45015/hbase/oldWALs, maxLogs=32 2024-12-03T15:57:33,288 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-82468117/hregion-82468117.1733241453275, exclude list is [], retry=0 2024-12-03T15:57:33,292 DEBUG [AsyncFSWAL-26-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44827,DS-e007f1cc-fc1b-4368-9716-e11e4fe2f4eb,DISK] 2024-12-03T15:57:33,294 DEBUG [AsyncFSWAL-26-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40575,DS-309fe9b7-1997-44cb-b0c9-1ce3c04a71de,DISK] 2024-12-03T15:57:33,302 DEBUG [AsyncFSWAL-26-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46585,DS-5c6771fd-75c7-4775-ad88-ae5bbe88de5c,DISK] 2024-12-03T15:57:33,346 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-82468117/hregion-82468117.1733241453275 2024-12-03T15:57:33,347 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46731:46731),(127.0.0.1/127.0.0.1:44513:44513),(127.0.0.1/127.0.0.1:37355:37355)] 2024-12-03T15:57:33,348 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 78064417e81f1d5c7da16577002b0ad5, NAME => 'testCompactedBulkLoadedFiles,,1733241453270.78064417e81f1d5c7da16577002b0ad5.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testCompactedBulkLoadedFiles', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45015/hbase 2024-12-03T15:57:33,372 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46585 is added to blk_1073741916_1094 (size=63) 2024-12-03T15:57:33,379 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40575 is added to blk_1073741916_1094 (size=63) 2024-12-03T15:57:33,379 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44827 is added to blk_1073741916_1094 (size=63) 2024-12-03T15:57:33,379 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testCompactedBulkLoadedFiles,,1733241453270.78064417e81f1d5c7da16577002b0ad5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T15:57:33,382 INFO [StoreOpener-78064417e81f1d5c7da16577002b0ad5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 78064417e81f1d5c7da16577002b0ad5 2024-12-03T15:57:33,384 INFO [StoreOpener-78064417e81f1d5c7da16577002b0ad5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 78064417e81f1d5c7da16577002b0ad5 columnFamilyName a 2024-12-03T15:57:33,384 DEBUG [StoreOpener-78064417e81f1d5c7da16577002b0ad5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T15:57:33,384 INFO [StoreOpener-78064417e81f1d5c7da16577002b0ad5-1 {}] regionserver.HStore(327): Store=78064417e81f1d5c7da16577002b0ad5/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T15:57:33,385 INFO [StoreOpener-78064417e81f1d5c7da16577002b0ad5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 78064417e81f1d5c7da16577002b0ad5 2024-12-03T15:57:33,386 INFO [StoreOpener-78064417e81f1d5c7da16577002b0ad5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 78064417e81f1d5c7da16577002b0ad5 columnFamilyName b 2024-12-03T15:57:33,386 DEBUG [StoreOpener-78064417e81f1d5c7da16577002b0ad5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T15:57:33,388 INFO [StoreOpener-78064417e81f1d5c7da16577002b0ad5-1 {}] regionserver.HStore(327): Store=78064417e81f1d5c7da16577002b0ad5/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T15:57:33,388 INFO [StoreOpener-78064417e81f1d5c7da16577002b0ad5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 78064417e81f1d5c7da16577002b0ad5 2024-12-03T15:57:33,389 INFO [StoreOpener-78064417e81f1d5c7da16577002b0ad5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 78064417e81f1d5c7da16577002b0ad5 columnFamilyName c 2024-12-03T15:57:33,390 DEBUG [StoreOpener-78064417e81f1d5c7da16577002b0ad5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T15:57:33,390 INFO [StoreOpener-78064417e81f1d5c7da16577002b0ad5-1 {}] regionserver.HStore(327): Store=78064417e81f1d5c7da16577002b0ad5/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T15:57:33,390 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 78064417e81f1d5c7da16577002b0ad5 2024-12-03T15:57:33,394 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45015/hbase/data/default/testCompactedBulkLoadedFiles/78064417e81f1d5c7da16577002b0ad5 2024-12-03T15:57:33,395 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45015/hbase/data/default/testCompactedBulkLoadedFiles/78064417e81f1d5c7da16577002b0ad5 2024-12-03T15:57:33,398 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 78064417e81f1d5c7da16577002b0ad5 2024-12-03T15:57:33,398 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 78064417e81f1d5c7da16577002b0ad5 2024-12-03T15:57:33,399 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testCompactedBulkLoadedFiles descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-03T15:57:33,400 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 78064417e81f1d5c7da16577002b0ad5 2024-12-03T15:57:33,407 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45015/hbase/data/default/testCompactedBulkLoadedFiles/78064417e81f1d5c7da16577002b0ad5/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T15:57:33,407 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 78064417e81f1d5c7da16577002b0ad5; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65439046, jitterRate=-0.024882227182388306}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-03T15:57:33,408 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 78064417e81f1d5c7da16577002b0ad5: Writing region info on filesystem at 1733241453380Initializing all the Stores at 1733241453381 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733241453381Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733241453382 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733241453382Cleaning up temporary data from old regions at 1733241453398 (+16 ms)Region opened successfully at 1733241453408 (+10 ms) 2024-12-03T15:57:33,409 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 78064417e81f1d5c7da16577002b0ad5, disabling compactions & flushes 2024-12-03T15:57:33,409 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testCompactedBulkLoadedFiles,,1733241453270.78064417e81f1d5c7da16577002b0ad5. 2024-12-03T15:57:33,409 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testCompactedBulkLoadedFiles,,1733241453270.78064417e81f1d5c7da16577002b0ad5. 2024-12-03T15:57:33,409 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testCompactedBulkLoadedFiles,,1733241453270.78064417e81f1d5c7da16577002b0ad5. after waiting 0 ms 2024-12-03T15:57:33,409 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testCompactedBulkLoadedFiles,,1733241453270.78064417e81f1d5c7da16577002b0ad5. 2024-12-03T15:57:33,414 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testCompactedBulkLoadedFiles,,1733241453270.78064417e81f1d5c7da16577002b0ad5. 2024-12-03T15:57:33,414 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 78064417e81f1d5c7da16577002b0ad5: Waiting for close lock at 1733241453409Disabling compacts and flushes for region at 1733241453409Disabling writes for close at 1733241453409Writing region close event to WAL at 1733241453414 (+5 ms)Closed at 1733241453414 2024-12-03T15:57:33,425 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44827 is added to blk_1073741915_1093 (size=175) 2024-12-03T15:57:33,426 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46585 is added to blk_1073741915_1093 (size=175) 2024-12-03T15:57:33,426 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40575 is added to blk_1073741915_1093 (size=175) 2024-12-03T15:57:33,430 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-03T15:57:33,430 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-82468117:(num 1733241453275) 2024-12-03T15:57:33,431 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-03T15:57:33,433 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:45015/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733241453269, archiveDir=hdfs://localhost:45015/hbase/oldWALs, maxLogs=32 2024-12-03T15:57:33,448 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733241453269/wal.1733241453433, exclude list is [], retry=0 2024-12-03T15:57:33,451 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44827,DS-e007f1cc-fc1b-4368-9716-e11e4fe2f4eb,DISK] 2024-12-03T15:57:33,452 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40575,DS-309fe9b7-1997-44cb-b0c9-1ce3c04a71de,DISK] 2024-12-03T15:57:33,452 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46585,DS-5c6771fd-75c7-4775-ad88-ae5bbe88de5c,DISK] 2024-12-03T15:57:33,484 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733241453269/wal.1733241453433 2024-12-03T15:57:33,486 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46731:46731),(127.0.0.1/127.0.0.1:44513:44513),(127.0.0.1/127.0.0.1:37355:37355)] 2024-12-03T15:57:33,486 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 78064417e81f1d5c7da16577002b0ad5, NAME => 'testCompactedBulkLoadedFiles,,1733241453270.78064417e81f1d5c7da16577002b0ad5.', STARTKEY => '', ENDKEY => ''} 2024-12-03T15:57:33,486 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testCompactedBulkLoadedFiles,,1733241453270.78064417e81f1d5c7da16577002b0ad5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T15:57:33,486 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 78064417e81f1d5c7da16577002b0ad5 2024-12-03T15:57:33,486 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 78064417e81f1d5c7da16577002b0ad5 2024-12-03T15:57:33,488 INFO [StoreOpener-78064417e81f1d5c7da16577002b0ad5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 78064417e81f1d5c7da16577002b0ad5 2024-12-03T15:57:33,489 INFO [StoreOpener-78064417e81f1d5c7da16577002b0ad5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 78064417e81f1d5c7da16577002b0ad5 columnFamilyName a 2024-12-03T15:57:33,489 DEBUG [StoreOpener-78064417e81f1d5c7da16577002b0ad5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T15:57:33,490 INFO [StoreOpener-78064417e81f1d5c7da16577002b0ad5-1 {}] regionserver.HStore(327): Store=78064417e81f1d5c7da16577002b0ad5/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T15:57:33,490 INFO [StoreOpener-78064417e81f1d5c7da16577002b0ad5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 78064417e81f1d5c7da16577002b0ad5 2024-12-03T15:57:33,490 INFO [StoreOpener-78064417e81f1d5c7da16577002b0ad5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 78064417e81f1d5c7da16577002b0ad5 columnFamilyName b 2024-12-03T15:57:33,491 DEBUG [StoreOpener-78064417e81f1d5c7da16577002b0ad5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T15:57:33,491 INFO [StoreOpener-78064417e81f1d5c7da16577002b0ad5-1 {}] regionserver.HStore(327): Store=78064417e81f1d5c7da16577002b0ad5/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T15:57:33,491 INFO [StoreOpener-78064417e81f1d5c7da16577002b0ad5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 78064417e81f1d5c7da16577002b0ad5 2024-12-03T15:57:33,492 INFO [StoreOpener-78064417e81f1d5c7da16577002b0ad5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 78064417e81f1d5c7da16577002b0ad5 columnFamilyName c 2024-12-03T15:57:33,492 DEBUG [StoreOpener-78064417e81f1d5c7da16577002b0ad5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T15:57:33,492 INFO [StoreOpener-78064417e81f1d5c7da16577002b0ad5-1 {}] regionserver.HStore(327): Store=78064417e81f1d5c7da16577002b0ad5/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T15:57:33,492 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 78064417e81f1d5c7da16577002b0ad5 2024-12-03T15:57:33,493 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45015/hbase/data/default/testCompactedBulkLoadedFiles/78064417e81f1d5c7da16577002b0ad5 2024-12-03T15:57:33,494 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45015/hbase/data/default/testCompactedBulkLoadedFiles/78064417e81f1d5c7da16577002b0ad5 2024-12-03T15:57:33,495 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 78064417e81f1d5c7da16577002b0ad5 2024-12-03T15:57:33,495 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 78064417e81f1d5c7da16577002b0ad5 2024-12-03T15:57:33,496 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testCompactedBulkLoadedFiles descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-03T15:57:33,502 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 78064417e81f1d5c7da16577002b0ad5 2024-12-03T15:57:33,503 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 78064417e81f1d5c7da16577002b0ad5; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75256420, jitterRate=0.12140804529190063}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-03T15:57:33,504 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 78064417e81f1d5c7da16577002b0ad5: Writing region info on filesystem at 1733241453487Initializing all the Stores at 1733241453488 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733241453488Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733241453488Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733241453488Cleaning up temporary data from old regions at 1733241453495 (+7 ms)Region opened successfully at 1733241453504 (+9 ms) 2024-12-03T15:57:33,511 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45015/hbase/testCompactedBulkLoadedFiles/hfile0 is 32, key is 000/a:a/1733241453511/Put/seqid=0 2024-12-03T15:57:33,531 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44827 is added to blk_1073741918_1096 (size=4875) 2024-12-03T15:57:33,531 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40575 is added to blk_1073741918_1096 (size=4875) 2024-12-03T15:57:33,531 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46585 is added to blk_1073741918_1096 (size=4875) 2024-12-03T15:57:33,936 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45015/hbase/testCompactedBulkLoadedFiles/hfile1 is 32, key is 100/a:a/1733241453935/Put/seqid=0 2024-12-03T15:57:33,954 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44827 is added to blk_1073741919_1097 (size=4875) 2024-12-03T15:57:33,954 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40575 is added to blk_1073741919_1097 (size=4875) 2024-12-03T15:57:33,955 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46585 is added to blk_1073741919_1097 (size=4875) 2024-12-03T15:57:33,958 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45015/hbase/testCompactedBulkLoadedFiles/hfile2 is 32, key is 200/a:a/1733241453957/Put/seqid=0 2024-12-03T15:57:33,980 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46585 is added to blk_1073741920_1098 (size=4875) 2024-12-03T15:57:33,982 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44827 is added to blk_1073741920_1098 (size=4875) 2024-12-03T15:57:33,982 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40575 is added to blk_1073741920_1098 (size=4875) 2024-12-03T15:57:33,982 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:45015/hbase/testCompactedBulkLoadedFiles/hfile0 for inclusion in 78064417e81f1d5c7da16577002b0ad5/a 2024-12-03T15:57:33,988 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first=000 last=050 2024-12-03T15:57:33,988 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-12-03T15:57:33,988 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:45015/hbase/testCompactedBulkLoadedFiles/hfile1 for inclusion in 78064417e81f1d5c7da16577002b0ad5/a 2024-12-03T15:57:33,992 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first=100 last=150 2024-12-03T15:57:33,993 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-12-03T15:57:33,993 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:45015/hbase/testCompactedBulkLoadedFiles/hfile2 for inclusion in 78064417e81f1d5c7da16577002b0ad5/a 2024-12-03T15:57:33,997 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first=200 last=250 2024-12-03T15:57:33,997 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-12-03T15:57:33,997 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 78064417e81f1d5c7da16577002b0ad5 3/3 column families, dataSize=51 B heapSize=896 B 2024-12-03T15:57:34,022 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45015/hbase/data/default/testCompactedBulkLoadedFiles/78064417e81f1d5c7da16577002b0ad5/.tmp/a/22c449edcf1f4072b154e4a7d70cfa77 is 55, key is testCompactedBulkLoadedFiles/a:a/1733241453504/Put/seqid=0 2024-12-03T15:57:34,030 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40575 is added to blk_1073741921_1099 (size=5107) 2024-12-03T15:57:34,031 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46585 is added to blk_1073741921_1099 (size=5107) 2024-12-03T15:57:34,032 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44827 is added to blk_1073741921_1099 (size=5107) 2024-12-03T15:57:34,032 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51 B at sequenceid=4 (bloomFilter=true), to=hdfs://localhost:45015/hbase/data/default/testCompactedBulkLoadedFiles/78064417e81f1d5c7da16577002b0ad5/.tmp/a/22c449edcf1f4072b154e4a7d70cfa77 2024-12-03T15:57:34,039 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45015/hbase/data/default/testCompactedBulkLoadedFiles/78064417e81f1d5c7da16577002b0ad5/.tmp/a/22c449edcf1f4072b154e4a7d70cfa77 as hdfs://localhost:45015/hbase/data/default/testCompactedBulkLoadedFiles/78064417e81f1d5c7da16577002b0ad5/a/22c449edcf1f4072b154e4a7d70cfa77 2024-12-03T15:57:34,044 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45015/hbase/data/default/testCompactedBulkLoadedFiles/78064417e81f1d5c7da16577002b0ad5/a/22c449edcf1f4072b154e4a7d70cfa77, entries=1, sequenceid=4, filesize=5.0 K 2024-12-03T15:57:34,046 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~51 B/51, heapSize ~368 B/368, currentSize=0 B/0 for 78064417e81f1d5c7da16577002b0ad5 in 49ms, sequenceid=4, compaction requested=false 2024-12-03T15:57:34,046 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 78064417e81f1d5c7da16577002b0ad5: 2024-12-03T15:57:34,048 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45015/hbase/testCompactedBulkLoadedFiles/hfile0 as hdfs://localhost:45015/hbase/data/default/testCompactedBulkLoadedFiles/78064417e81f1d5c7da16577002b0ad5/a/728da4ae986b4123b070cbab573639c5_SeqId_4_ 2024-12-03T15:57:34,049 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45015/hbase/testCompactedBulkLoadedFiles/hfile1 as hdfs://localhost:45015/hbase/data/default/testCompactedBulkLoadedFiles/78064417e81f1d5c7da16577002b0ad5/a/0b94c15f16b84928aa631df392cbe32a_SeqId_4_ 2024-12-03T15:57:34,050 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45015/hbase/testCompactedBulkLoadedFiles/hfile2 as hdfs://localhost:45015/hbase/data/default/testCompactedBulkLoadedFiles/78064417e81f1d5c7da16577002b0ad5/a/1d9d6ea2c6224e488ba2db4448846b3a_SeqId_4_ 2024-12-03T15:57:34,051 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:45015/hbase/testCompactedBulkLoadedFiles/hfile0 into 78064417e81f1d5c7da16577002b0ad5/a as hdfs://localhost:45015/hbase/data/default/testCompactedBulkLoadedFiles/78064417e81f1d5c7da16577002b0ad5/a/728da4ae986b4123b070cbab573639c5_SeqId_4_ - updating store file list. 2024-12-03T15:57:34,057 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for 728da4ae986b4123b070cbab573639c5_SeqId_4_: NONE, but ROW specified in column family configuration 2024-12-03T15:57:34,057 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:45015/hbase/data/default/testCompactedBulkLoadedFiles/78064417e81f1d5c7da16577002b0ad5/a/728da4ae986b4123b070cbab573639c5_SeqId_4_ into 78064417e81f1d5c7da16577002b0ad5/a 2024-12-03T15:57:34,057 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:45015/hbase/testCompactedBulkLoadedFiles/hfile0 into 78064417e81f1d5c7da16577002b0ad5/a (new location: hdfs://localhost:45015/hbase/data/default/testCompactedBulkLoadedFiles/78064417e81f1d5c7da16577002b0ad5/a/728da4ae986b4123b070cbab573639c5_SeqId_4_) 2024-12-03T15:57:34,059 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:45015/hbase/testCompactedBulkLoadedFiles/hfile1 into 78064417e81f1d5c7da16577002b0ad5/a as hdfs://localhost:45015/hbase/data/default/testCompactedBulkLoadedFiles/78064417e81f1d5c7da16577002b0ad5/a/0b94c15f16b84928aa631df392cbe32a_SeqId_4_ - updating store file list. 2024-12-03T15:57:34,064 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for 0b94c15f16b84928aa631df392cbe32a_SeqId_4_: NONE, but ROW specified in column family configuration 2024-12-03T15:57:34,064 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:45015/hbase/data/default/testCompactedBulkLoadedFiles/78064417e81f1d5c7da16577002b0ad5/a/0b94c15f16b84928aa631df392cbe32a_SeqId_4_ into 78064417e81f1d5c7da16577002b0ad5/a 2024-12-03T15:57:34,064 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:45015/hbase/testCompactedBulkLoadedFiles/hfile1 into 78064417e81f1d5c7da16577002b0ad5/a (new location: hdfs://localhost:45015/hbase/data/default/testCompactedBulkLoadedFiles/78064417e81f1d5c7da16577002b0ad5/a/0b94c15f16b84928aa631df392cbe32a_SeqId_4_) 2024-12-03T15:57:34,066 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:45015/hbase/testCompactedBulkLoadedFiles/hfile2 into 78064417e81f1d5c7da16577002b0ad5/a as hdfs://localhost:45015/hbase/data/default/testCompactedBulkLoadedFiles/78064417e81f1d5c7da16577002b0ad5/a/1d9d6ea2c6224e488ba2db4448846b3a_SeqId_4_ - updating store file list. 2024-12-03T15:57:34,071 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for 1d9d6ea2c6224e488ba2db4448846b3a_SeqId_4_: NONE, but ROW specified in column family configuration 2024-12-03T15:57:34,071 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:45015/hbase/data/default/testCompactedBulkLoadedFiles/78064417e81f1d5c7da16577002b0ad5/a/1d9d6ea2c6224e488ba2db4448846b3a_SeqId_4_ into 78064417e81f1d5c7da16577002b0ad5/a 2024-12-03T15:57:34,071 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:45015/hbase/testCompactedBulkLoadedFiles/hfile2 into 78064417e81f1d5c7da16577002b0ad5/a (new location: hdfs://localhost:45015/hbase/data/default/testCompactedBulkLoadedFiles/78064417e81f1d5c7da16577002b0ad5/a/1d9d6ea2c6224e488ba2db4448846b3a_SeqId_4_) 2024-12-03T15:57:34,082 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T15:57:34,082 DEBUG [Time-limited test {}] regionserver.HStore(1541): 78064417e81f1d5c7da16577002b0ad5/a is initiating major compaction (all files) 2024-12-03T15:57:34,082 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of 78064417e81f1d5c7da16577002b0ad5/a in testCompactedBulkLoadedFiles,,1733241453270.78064417e81f1d5c7da16577002b0ad5. 2024-12-03T15:57:34,082 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45015/hbase/data/default/testCompactedBulkLoadedFiles/78064417e81f1d5c7da16577002b0ad5/a/22c449edcf1f4072b154e4a7d70cfa77, hdfs://localhost:45015/hbase/data/default/testCompactedBulkLoadedFiles/78064417e81f1d5c7da16577002b0ad5/a/728da4ae986b4123b070cbab573639c5_SeqId_4_, hdfs://localhost:45015/hbase/data/default/testCompactedBulkLoadedFiles/78064417e81f1d5c7da16577002b0ad5/a/0b94c15f16b84928aa631df392cbe32a_SeqId_4_, hdfs://localhost:45015/hbase/data/default/testCompactedBulkLoadedFiles/78064417e81f1d5c7da16577002b0ad5/a/1d9d6ea2c6224e488ba2db4448846b3a_SeqId_4_] into tmpdir=hdfs://localhost:45015/hbase/data/default/testCompactedBulkLoadedFiles/78064417e81f1d5c7da16577002b0ad5/.tmp, totalSize=19.3 K 2024-12-03T15:57:34,083 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 22c449edcf1f4072b154e4a7d70cfa77, keycount=1, bloomtype=ROW, size=5.0 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=1733241453504 2024-12-03T15:57:34,084 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 728da4ae986b4123b070cbab573639c5_SeqId_4_, keycount=10, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=-9223372036854775808 2024-12-03T15:57:34,084 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 0b94c15f16b84928aa631df392cbe32a_SeqId_4_, keycount=10, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=-9223372036854775808 2024-12-03T15:57:34,085 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 1d9d6ea2c6224e488ba2db4448846b3a_SeqId_4_, keycount=10, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=-9223372036854775808 2024-12-03T15:57:34,099 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45015/hbase/data/default/testCompactedBulkLoadedFiles/78064417e81f1d5c7da16577002b0ad5/.tmp/a/913c086bba08457b968802a2d04e2431 is 55, key is testCompactedBulkLoadedFiles/a:a/1733241453504/Put/seqid=0 2024-12-03T15:57:34,112 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40575 is added to blk_1073741922_1100 (size=6154) 2024-12-03T15:57:34,113 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46585 is added to blk_1073741922_1100 (size=6154) 2024-12-03T15:57:34,113 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44827 is added to blk_1073741922_1100 (size=6154) 2024-12-03T15:57:34,121 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45015/hbase/data/default/testCompactedBulkLoadedFiles/78064417e81f1d5c7da16577002b0ad5/.tmp/a/913c086bba08457b968802a2d04e2431 as hdfs://localhost:45015/hbase/data/default/testCompactedBulkLoadedFiles/78064417e81f1d5c7da16577002b0ad5/a/913c086bba08457b968802a2d04e2431 2024-12-03T15:57:34,128 INFO [Time-limited test {}] regionserver.HStore(1337): Completed major compaction of 4 (all) file(s) in 78064417e81f1d5c7da16577002b0ad5/a of 78064417e81f1d5c7da16577002b0ad5 into 913c086bba08457b968802a2d04e2431(size=6.0 K), total size for store is 6.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T15:57:34,128 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for 78064417e81f1d5c7da16577002b0ad5: 2024-12-03T15:57:34,128 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 0 store files, 0 compacting, 0 eligible, 16 blocking 2024-12-03T15:57:34,128 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 0 store files, 0 compacting, 0 eligible, 16 blocking 2024-12-03T15:57:34,165 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:45015/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733241453269/wal.1733241453433, size=0 (0bytes) 2024-12-03T15:57:34,166 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:45015/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733241453269/wal.1733241453433 might be still open, length is 0 2024-12-03T15:57:34,166 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:45015/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733241453269/wal.1733241453433 2024-12-03T15:57:34,166 WARN [IPC Server handler 0 on default port 45015 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733241453269/wal.1733241453433 has not been closed. Lease recovery is in progress. RecoveryId = 1101 for block blk_1073741917_1095 2024-12-03T15:57:34,167 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:45015/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733241453269/wal.1733241453433 after 1ms 2024-12-03T15:57:35,067 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-365845159_22 at /127.0.0.1:35314 [Receiving block BP-1122350750-172.17.0.3-1733241413817:blk_1073741917_1095] {}] datanode.DataXceiver(331): 127.0.0.1:44827:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:35314 dst: /127.0.0.1:44827 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:44827 remote=/127.0.0.1:35314]. Total timeout mills is 60000, 59062 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T15:57:35,068 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-365845159_22 at /127.0.0.1:60830 [Receiving block BP-1122350750-172.17.0.3-1733241413817:blk_1073741917_1095] {}] datanode.DataXceiver(331): 127.0.0.1:46585:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:60830 dst: /127.0.0.1:46585 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T15:57:35,069 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-365845159_22 at /127.0.0.1:41922 [Receiving block BP-1122350750-172.17.0.3-1733241413817:blk_1073741917_1095] {}] datanode.DataXceiver(331): 127.0.0.1:40575:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:41922 dst: /127.0.0.1:40575 java.io.InterruptedIOException: Interrupted receiveBlock at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T15:57:35,073 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44827 is added to blk_1073741917_1101 (size=1794) 2024-12-03T15:57:35,073 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40575 is added to blk_1073741917_1101 (size=1794) 2024-12-03T15:57:37,167 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-03T15:57:37,779 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testDatalossWhenInputError 2024-12-03T15:57:37,779 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testDatalossWhenInputError Metrics about Tables on a single HBase RegionServer 2024-12-03T15:57:37,780 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testCompactedBulkLoadedFiles 2024-12-03T15:57:37,780 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testCompactedBulkLoadedFiles Metrics about Tables on a single HBase RegionServer 2024-12-03T15:57:38,168 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:45015/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733241453269/wal.1733241453433 after 4001ms 2024-12-03T15:57:38,210 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-03T15:57:38,210 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:45015/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733241453269/wal.1733241453433 took 4045ms 2024-12-03T15:57:38,212 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:45015/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733241453269/wal.1733241453433; continuing. 2024-12-03T15:57:38,213 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:45015/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733241453269/wal.1733241453433 so closing down 2024-12-03T15:57:38,213 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-03T15:57:38,215 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1733241453433.temp 2024-12-03T15:57:38,256 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:45015/hbase/data/default/testCompactedBulkLoadedFiles/78064417e81f1d5c7da16577002b0ad5/recovered.edits/0000000000000000003-wal.1733241453433.temp 2024-12-03T15:57:38,256 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-03T15:57:38,262 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44827 is added to blk_1073741923_1102 (size=800) 2024-12-03T15:57:38,263 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46585 is added to blk_1073741923_1102 (size=800) 2024-12-03T15:57:38,263 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40575 is added to blk_1073741923_1102 (size=800) 2024-12-03T15:57:38,264 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:45015/hbase/data/default/testCompactedBulkLoadedFiles/78064417e81f1d5c7da16577002b0ad5/recovered.edits/0000000000000000003-wal.1733241453433.temp (wrote 2 edits, skipped 0 edits in 0 ms) 2024-12-03T15:57:38,265 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:45015/hbase/data/default/testCompactedBulkLoadedFiles/78064417e81f1d5c7da16577002b0ad5/recovered.edits/0000000000000000003-wal.1733241453433.temp to hdfs://localhost:45015/hbase/data/default/testCompactedBulkLoadedFiles/78064417e81f1d5c7da16577002b0ad5/recovered.edits/0000000000000000008 2024-12-03T15:57:38,265 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 5 edits across 1 Regions in 55 ms; skipped=3; WAL=hdfs://localhost:45015/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733241453269/wal.1733241453433, size=0, length=0, corrupted=false, cancelled=false 2024-12-03T15:57:38,265 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:45015/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733241453269/wal.1733241453433, journal: Splitting hdfs://localhost:45015/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733241453269/wal.1733241453433, size=0 (0bytes) at 1733241454165Finishing writing output for hdfs://localhost:45015/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733241453269/wal.1733241453433 so closing down at 1733241458213 (+4048 ms)Creating recovered edits writer path=hdfs://localhost:45015/hbase/data/default/testCompactedBulkLoadedFiles/78064417e81f1d5c7da16577002b0ad5/recovered.edits/0000000000000000003-wal.1733241453433.temp at 1733241458256 (+43 ms)3 split writer threads finished at 1733241458256Closed recovered edits writer path=hdfs://localhost:45015/hbase/data/default/testCompactedBulkLoadedFiles/78064417e81f1d5c7da16577002b0ad5/recovered.edits/0000000000000000003-wal.1733241453433.temp (wrote 2 edits, skipped 0 edits in 0 ms) at 1733241458264 (+8 ms)Rename recovered edits hdfs://localhost:45015/hbase/data/default/testCompactedBulkLoadedFiles/78064417e81f1d5c7da16577002b0ad5/recovered.edits/0000000000000000003-wal.1733241453433.temp to hdfs://localhost:45015/hbase/data/default/testCompactedBulkLoadedFiles/78064417e81f1d5c7da16577002b0ad5/recovered.edits/0000000000000000008 at 1733241458265 (+1 ms)Processed 5 edits across 1 Regions in 55 ms; skipped=3; WAL=hdfs://localhost:45015/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733241453269/wal.1733241453433, size=0, length=0, corrupted=false, cancelled=false at 1733241458265 2024-12-03T15:57:38,267 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:45015/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733241453269/wal.1733241453433 to hdfs://localhost:45015/hbase/oldWALs/wal.1733241453433 2024-12-03T15:57:38,268 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:45015/hbase/data/default/testCompactedBulkLoadedFiles/78064417e81f1d5c7da16577002b0ad5/recovered.edits/0000000000000000008 2024-12-03T15:57:38,268 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-03T15:57:38,270 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:45015/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733241453269, archiveDir=hdfs://localhost:45015/hbase/oldWALs, maxLogs=32 2024-12-03T15:57:38,291 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733241453269/wal.1733241458270, exclude list is [], retry=0 2024-12-03T15:57:38,294 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46585,DS-5c6771fd-75c7-4775-ad88-ae5bbe88de5c,DISK] 2024-12-03T15:57:38,294 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40575,DS-309fe9b7-1997-44cb-b0c9-1ce3c04a71de,DISK] 2024-12-03T15:57:38,294 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44827,DS-e007f1cc-fc1b-4368-9716-e11e4fe2f4eb,DISK] 2024-12-03T15:57:38,321 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733241453269/wal.1733241458270 2024-12-03T15:57:38,322 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37355:37355),(127.0.0.1/127.0.0.1:44513:44513),(127.0.0.1/127.0.0.1:46731:46731)] 2024-12-03T15:57:38,322 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 78064417e81f1d5c7da16577002b0ad5, NAME => 'testCompactedBulkLoadedFiles,,1733241453270.78064417e81f1d5c7da16577002b0ad5.', STARTKEY => '', ENDKEY => ''} 2024-12-03T15:57:38,322 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testCompactedBulkLoadedFiles,,1733241453270.78064417e81f1d5c7da16577002b0ad5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T15:57:38,322 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 78064417e81f1d5c7da16577002b0ad5 2024-12-03T15:57:38,322 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 78064417e81f1d5c7da16577002b0ad5 2024-12-03T15:57:38,324 INFO [StoreOpener-78064417e81f1d5c7da16577002b0ad5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 78064417e81f1d5c7da16577002b0ad5 2024-12-03T15:57:38,325 INFO [StoreOpener-78064417e81f1d5c7da16577002b0ad5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 78064417e81f1d5c7da16577002b0ad5 columnFamilyName a 2024-12-03T15:57:38,325 DEBUG [StoreOpener-78064417e81f1d5c7da16577002b0ad5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T15:57:38,331 DEBUG [StoreFileOpener-78064417e81f1d5c7da16577002b0ad5-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for 0b94c15f16b84928aa631df392cbe32a_SeqId_4_: NONE, but ROW specified in column family configuration 2024-12-03T15:57:38,331 DEBUG [StoreOpener-78064417e81f1d5c7da16577002b0ad5-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:45015/hbase/data/default/testCompactedBulkLoadedFiles/78064417e81f1d5c7da16577002b0ad5/a/0b94c15f16b84928aa631df392cbe32a_SeqId_4_ 2024-12-03T15:57:38,335 DEBUG [StoreFileOpener-78064417e81f1d5c7da16577002b0ad5-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for 1d9d6ea2c6224e488ba2db4448846b3a_SeqId_4_: NONE, but ROW specified in column family configuration 2024-12-03T15:57:38,335 DEBUG [StoreOpener-78064417e81f1d5c7da16577002b0ad5-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:45015/hbase/data/default/testCompactedBulkLoadedFiles/78064417e81f1d5c7da16577002b0ad5/a/1d9d6ea2c6224e488ba2db4448846b3a_SeqId_4_ 2024-12-03T15:57:38,338 DEBUG [StoreOpener-78064417e81f1d5c7da16577002b0ad5-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:45015/hbase/data/default/testCompactedBulkLoadedFiles/78064417e81f1d5c7da16577002b0ad5/a/22c449edcf1f4072b154e4a7d70cfa77 2024-12-03T15:57:38,342 DEBUG [StoreFileOpener-78064417e81f1d5c7da16577002b0ad5-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for 728da4ae986b4123b070cbab573639c5_SeqId_4_: NONE, but ROW specified in column family configuration 2024-12-03T15:57:38,342 DEBUG [StoreOpener-78064417e81f1d5c7da16577002b0ad5-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:45015/hbase/data/default/testCompactedBulkLoadedFiles/78064417e81f1d5c7da16577002b0ad5/a/728da4ae986b4123b070cbab573639c5_SeqId_4_ 2024-12-03T15:57:38,346 DEBUG [StoreOpener-78064417e81f1d5c7da16577002b0ad5-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:45015/hbase/data/default/testCompactedBulkLoadedFiles/78064417e81f1d5c7da16577002b0ad5/a/913c086bba08457b968802a2d04e2431 2024-12-03T15:57:38,346 WARN [StoreOpener-78064417e81f1d5c7da16577002b0ad5-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:45015/hbase/data/default/testCompactedBulkLoadedFiles/78064417e81f1d5c7da16577002b0ad5/a/0b94c15f16b84928aa631df392cbe32a_SeqId_4_ from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@ee55e31 2024-12-03T15:57:38,346 WARN [StoreOpener-78064417e81f1d5c7da16577002b0ad5-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:45015/hbase/data/default/testCompactedBulkLoadedFiles/78064417e81f1d5c7da16577002b0ad5/a/1d9d6ea2c6224e488ba2db4448846b3a_SeqId_4_ from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@ee55e31 2024-12-03T15:57:38,346 WARN [StoreOpener-78064417e81f1d5c7da16577002b0ad5-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:45015/hbase/data/default/testCompactedBulkLoadedFiles/78064417e81f1d5c7da16577002b0ad5/a/22c449edcf1f4072b154e4a7d70cfa77 from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@ee55e31 2024-12-03T15:57:38,346 WARN [StoreOpener-78064417e81f1d5c7da16577002b0ad5-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:45015/hbase/data/default/testCompactedBulkLoadedFiles/78064417e81f1d5c7da16577002b0ad5/a/728da4ae986b4123b070cbab573639c5_SeqId_4_ from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@ee55e31 2024-12-03T15:57:38,346 DEBUG [StoreOpener-78064417e81f1d5c7da16577002b0ad5-1 {}] regionserver.StoreEngine(327): Moving the files [hdfs://localhost:45015/hbase/data/default/testCompactedBulkLoadedFiles/78064417e81f1d5c7da16577002b0ad5/a/0b94c15f16b84928aa631df392cbe32a_SeqId_4_, hdfs://localhost:45015/hbase/data/default/testCompactedBulkLoadedFiles/78064417e81f1d5c7da16577002b0ad5/a/1d9d6ea2c6224e488ba2db4448846b3a_SeqId_4_, hdfs://localhost:45015/hbase/data/default/testCompactedBulkLoadedFiles/78064417e81f1d5c7da16577002b0ad5/a/22c449edcf1f4072b154e4a7d70cfa77, hdfs://localhost:45015/hbase/data/default/testCompactedBulkLoadedFiles/78064417e81f1d5c7da16577002b0ad5/a/728da4ae986b4123b070cbab573639c5_SeqId_4_] to archive 2024-12-03T15:57:38,347 DEBUG [StoreOpener-78064417e81f1d5c7da16577002b0ad5-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-03T15:57:38,348 DEBUG [StoreOpener-78064417e81f1d5c7da16577002b0ad5-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45015/hbase/data/default/testCompactedBulkLoadedFiles/78064417e81f1d5c7da16577002b0ad5/a/0b94c15f16b84928aa631df392cbe32a_SeqId_4_ to hdfs://localhost:45015/hbase/archive/data/default/testCompactedBulkLoadedFiles/78064417e81f1d5c7da16577002b0ad5/a/0b94c15f16b84928aa631df392cbe32a_SeqId_4_ 2024-12-03T15:57:38,350 DEBUG [StoreOpener-78064417e81f1d5c7da16577002b0ad5-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45015/hbase/data/default/testCompactedBulkLoadedFiles/78064417e81f1d5c7da16577002b0ad5/a/1d9d6ea2c6224e488ba2db4448846b3a_SeqId_4_ to hdfs://localhost:45015/hbase/archive/data/default/testCompactedBulkLoadedFiles/78064417e81f1d5c7da16577002b0ad5/a/1d9d6ea2c6224e488ba2db4448846b3a_SeqId_4_ 2024-12-03T15:57:38,351 DEBUG [StoreOpener-78064417e81f1d5c7da16577002b0ad5-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45015/hbase/data/default/testCompactedBulkLoadedFiles/78064417e81f1d5c7da16577002b0ad5/a/22c449edcf1f4072b154e4a7d70cfa77 to hdfs://localhost:45015/hbase/archive/data/default/testCompactedBulkLoadedFiles/78064417e81f1d5c7da16577002b0ad5/a/22c449edcf1f4072b154e4a7d70cfa77 2024-12-03T15:57:38,352 DEBUG [StoreOpener-78064417e81f1d5c7da16577002b0ad5-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45015/hbase/data/default/testCompactedBulkLoadedFiles/78064417e81f1d5c7da16577002b0ad5/a/728da4ae986b4123b070cbab573639c5_SeqId_4_ to hdfs://localhost:45015/hbase/archive/data/default/testCompactedBulkLoadedFiles/78064417e81f1d5c7da16577002b0ad5/a/728da4ae986b4123b070cbab573639c5_SeqId_4_ 2024-12-03T15:57:38,352 INFO [StoreOpener-78064417e81f1d5c7da16577002b0ad5-1 {}] regionserver.HStore(327): Store=78064417e81f1d5c7da16577002b0ad5/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T15:57:38,352 INFO [StoreOpener-78064417e81f1d5c7da16577002b0ad5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 78064417e81f1d5c7da16577002b0ad5 2024-12-03T15:57:38,353 INFO [StoreOpener-78064417e81f1d5c7da16577002b0ad5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 78064417e81f1d5c7da16577002b0ad5 columnFamilyName b 2024-12-03T15:57:38,353 DEBUG [StoreOpener-78064417e81f1d5c7da16577002b0ad5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T15:57:38,353 INFO [StoreOpener-78064417e81f1d5c7da16577002b0ad5-1 {}] regionserver.HStore(327): Store=78064417e81f1d5c7da16577002b0ad5/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T15:57:38,353 INFO [StoreOpener-78064417e81f1d5c7da16577002b0ad5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 78064417e81f1d5c7da16577002b0ad5 2024-12-03T15:57:38,354 INFO [StoreOpener-78064417e81f1d5c7da16577002b0ad5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 78064417e81f1d5c7da16577002b0ad5 columnFamilyName c 2024-12-03T15:57:38,354 DEBUG [StoreOpener-78064417e81f1d5c7da16577002b0ad5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T15:57:38,355 INFO [StoreOpener-78064417e81f1d5c7da16577002b0ad5-1 {}] regionserver.HStore(327): Store=78064417e81f1d5c7da16577002b0ad5/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T15:57:38,355 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 78064417e81f1d5c7da16577002b0ad5 2024-12-03T15:57:38,355 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45015/hbase/data/default/testCompactedBulkLoadedFiles/78064417e81f1d5c7da16577002b0ad5 2024-12-03T15:57:38,357 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:45015/hbase/data/default/testCompactedBulkLoadedFiles/78064417e81f1d5c7da16577002b0ad5 2024-12-03T15:57:38,357 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:45015/hbase/data/default/testCompactedBulkLoadedFiles/78064417e81f1d5c7da16577002b0ad5/recovered.edits/0000000000000000008 2024-12-03T15:57:38,385 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-03T15:57:38,387 DEBUG [Time-limited test {}] regionserver.HRegion(5836): 78064417e81f1d5c7da16577002b0ad5 : Replaying compaction marker table_name: "testCompactedBulkLoadedFiles" encoded_region_name: "78064417e81f1d5c7da16577002b0ad5" family_name: "a" compaction_input: "22c449edcf1f4072b154e4a7d70cfa77" compaction_input: "728da4ae986b4123b070cbab573639c5_SeqId_4_" compaction_input: "0b94c15f16b84928aa631df392cbe32a_SeqId_4_" compaction_input: "1d9d6ea2c6224e488ba2db4448846b3a_SeqId_4_" compaction_output: "913c086bba08457b968802a2d04e2431" store_home_dir: "a" region_name: "testCompactedBulkLoadedFiles,,1733241453270.78064417e81f1d5c7da16577002b0ad5." with seqId=9223372036854775807 and lastReplayedOpenRegionSeqId=-1 2024-12-03T15:57:38,387 DEBUG [Time-limited test {}] regionserver.HStore(1354): Completing compaction from the WAL marker 2024-12-03T15:57:38,387 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 0, skipped 2, firstSequenceIdInLog=3, maxSequenceIdInLog=8, path=hdfs://localhost:45015/hbase/data/default/testCompactedBulkLoadedFiles/78064417e81f1d5c7da16577002b0ad5/recovered.edits/0000000000000000008 2024-12-03T15:57:38,388 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:45015/hbase/data/default/testCompactedBulkLoadedFiles/78064417e81f1d5c7da16577002b0ad5/recovered.edits/0000000000000000008 2024-12-03T15:57:38,389 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 78064417e81f1d5c7da16577002b0ad5 2024-12-03T15:57:38,389 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 78064417e81f1d5c7da16577002b0ad5 2024-12-03T15:57:38,390 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testCompactedBulkLoadedFiles descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-03T15:57:38,392 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 78064417e81f1d5c7da16577002b0ad5 2024-12-03T15:57:38,394 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45015/hbase/data/default/testCompactedBulkLoadedFiles/78064417e81f1d5c7da16577002b0ad5/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-12-03T15:57:38,395 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 78064417e81f1d5c7da16577002b0ad5; next sequenceid=9; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70178341, jitterRate=0.04573877155780792}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-03T15:57:38,396 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 78064417e81f1d5c7da16577002b0ad5: Writing region info on filesystem at 1733241458322Initializing all the Stores at 1733241458323 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733241458323Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733241458323Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733241458323Cleaning up temporary data from old regions at 1733241458389 (+66 ms)Region opened successfully at 1733241458396 (+7 ms) 2024-12-03T15:57:38,398 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 78064417e81f1d5c7da16577002b0ad5, disabling compactions & flushes 2024-12-03T15:57:38,398 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testCompactedBulkLoadedFiles,,1733241453270.78064417e81f1d5c7da16577002b0ad5. 2024-12-03T15:57:38,398 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testCompactedBulkLoadedFiles,,1733241453270.78064417e81f1d5c7da16577002b0ad5. 2024-12-03T15:57:38,398 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testCompactedBulkLoadedFiles,,1733241453270.78064417e81f1d5c7da16577002b0ad5. after waiting 0 ms 2024-12-03T15:57:38,398 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testCompactedBulkLoadedFiles,,1733241453270.78064417e81f1d5c7da16577002b0ad5. 2024-12-03T15:57:38,399 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testCompactedBulkLoadedFiles,,1733241453270.78064417e81f1d5c7da16577002b0ad5. 2024-12-03T15:57:38,399 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 78064417e81f1d5c7da16577002b0ad5: Waiting for close lock at 1733241458398Disabling compacts and flushes for region at 1733241458398Disabling writes for close at 1733241458398Writing region close event to WAL at 1733241458399 (+1 ms)Closed at 1733241458399 2024-12-03T15:57:38,403 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46585 is added to blk_1073741924_1103 (size=175) 2024-12-03T15:57:38,404 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44827 is added to blk_1073741924_1103 (size=175) 2024-12-03T15:57:38,404 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40575 is added to blk_1073741924_1103 (size=175) 2024-12-03T15:57:38,406 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-03T15:57:38,406 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1733241458270) 2024-12-03T15:57:38,424 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testCompactedBulkLoadedFiles Thread=440 (was 429) Potentially hanging thread: LeaseRenewer:jenkinstestCompactedBulkLoadedFiles@localhost:45015 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-26-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-26-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-536404530_22 at /127.0.0.1:60904 [Waiting for operation #5] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-536404530_22 at /127.0.0.1:35368 [Waiting for operation #11] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-26-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1616464420) connection to localhost/127.0.0.1:45015 from jenkinstestCompactedBulkLoadedFiles java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-536404530_22 at /127.0.0.1:42012 [Waiting for operation #3] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1348 (was 1268) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=746 (was 776), ProcessCount=11 (was 11), AvailableMemoryMB=4897 (was 4854) - AvailableMemoryMB LEAK? - 2024-12-03T15:57:38,424 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1348 is superior to 1024 2024-12-03T15:57:38,438 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsWrittenViaHRegion Thread=440, OpenFileDescriptor=1348, MaxFileDescriptor=1048576, SystemLoadAverage=746, ProcessCount=11, AvailableMemoryMB=4895 2024-12-03T15:57:38,438 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1348 is superior to 1024 2024-12-03T15:57:38,453 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-03T15:57:38,455 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-03T15:57:38,455 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-03T15:57:38,458 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-71148152, suffix=, logDir=hdfs://localhost:45015/hbase/WALs/hregion-71148152, archiveDir=hdfs://localhost:45015/hbase/oldWALs, maxLogs=32 2024-12-03T15:57:38,472 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-71148152/hregion-71148152.1733241458458, exclude list is [], retry=0 2024-12-03T15:57:38,475 DEBUG [AsyncFSWAL-28-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40575,DS-309fe9b7-1997-44cb-b0c9-1ce3c04a71de,DISK] 2024-12-03T15:57:38,475 DEBUG [AsyncFSWAL-28-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44827,DS-e007f1cc-fc1b-4368-9716-e11e4fe2f4eb,DISK] 2024-12-03T15:57:38,475 DEBUG [AsyncFSWAL-28-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46585,DS-5c6771fd-75c7-4775-ad88-ae5bbe88de5c,DISK] 2024-12-03T15:57:38,516 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-71148152/hregion-71148152.1733241458458 2024-12-03T15:57:38,516 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44513:44513),(127.0.0.1/127.0.0.1:46731:46731),(127.0.0.1/127.0.0.1:37355:37355)] 2024-12-03T15:57:38,516 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 20f1b3d193b989a09e21a98de0f761b7, NAME => 'testReplayEditsWrittenViaHRegion,,1733241458453.20f1b3d193b989a09e21a98de0f761b7.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenViaHRegion', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45015/hbase 2024-12-03T15:57:38,525 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40575 is added to blk_1073741926_1105 (size=67) 2024-12-03T15:57:38,526 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46585 is added to blk_1073741926_1105 (size=67) 2024-12-03T15:57:38,526 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44827 is added to blk_1073741926_1105 (size=67) 2024-12-03T15:57:38,527 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1733241458453.20f1b3d193b989a09e21a98de0f761b7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T15:57:38,528 INFO [StoreOpener-20f1b3d193b989a09e21a98de0f761b7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 20f1b3d193b989a09e21a98de0f761b7 2024-12-03T15:57:38,529 INFO [StoreOpener-20f1b3d193b989a09e21a98de0f761b7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 20f1b3d193b989a09e21a98de0f761b7 columnFamilyName a 2024-12-03T15:57:38,529 DEBUG [StoreOpener-20f1b3d193b989a09e21a98de0f761b7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T15:57:38,530 INFO [StoreOpener-20f1b3d193b989a09e21a98de0f761b7-1 {}] regionserver.HStore(327): Store=20f1b3d193b989a09e21a98de0f761b7/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T15:57:38,530 INFO [StoreOpener-20f1b3d193b989a09e21a98de0f761b7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 20f1b3d193b989a09e21a98de0f761b7 2024-12-03T15:57:38,531 INFO [StoreOpener-20f1b3d193b989a09e21a98de0f761b7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 20f1b3d193b989a09e21a98de0f761b7 columnFamilyName b 2024-12-03T15:57:38,531 DEBUG [StoreOpener-20f1b3d193b989a09e21a98de0f761b7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T15:57:38,532 INFO [StoreOpener-20f1b3d193b989a09e21a98de0f761b7-1 {}] regionserver.HStore(327): Store=20f1b3d193b989a09e21a98de0f761b7/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T15:57:38,532 INFO [StoreOpener-20f1b3d193b989a09e21a98de0f761b7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 20f1b3d193b989a09e21a98de0f761b7 2024-12-03T15:57:38,533 INFO [StoreOpener-20f1b3d193b989a09e21a98de0f761b7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 20f1b3d193b989a09e21a98de0f761b7 columnFamilyName c 2024-12-03T15:57:38,533 DEBUG [StoreOpener-20f1b3d193b989a09e21a98de0f761b7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T15:57:38,534 INFO [StoreOpener-20f1b3d193b989a09e21a98de0f761b7-1 {}] regionserver.HStore(327): Store=20f1b3d193b989a09e21a98de0f761b7/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T15:57:38,534 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 20f1b3d193b989a09e21a98de0f761b7 2024-12-03T15:57:38,535 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45015/hbase/data/default/testReplayEditsWrittenViaHRegion/20f1b3d193b989a09e21a98de0f761b7 2024-12-03T15:57:38,535 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45015/hbase/data/default/testReplayEditsWrittenViaHRegion/20f1b3d193b989a09e21a98de0f761b7 2024-12-03T15:57:38,536 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 20f1b3d193b989a09e21a98de0f761b7 2024-12-03T15:57:38,536 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 20f1b3d193b989a09e21a98de0f761b7 2024-12-03T15:57:38,537 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-03T15:57:38,538 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 20f1b3d193b989a09e21a98de0f761b7 2024-12-03T15:57:38,542 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45015/hbase/data/default/testReplayEditsWrittenViaHRegion/20f1b3d193b989a09e21a98de0f761b7/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T15:57:38,542 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 20f1b3d193b989a09e21a98de0f761b7; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73266738, jitterRate=0.09175947308540344}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-03T15:57:38,542 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 20f1b3d193b989a09e21a98de0f761b7: Writing region info on filesystem at 1733241458527Initializing all the Stores at 1733241458528 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733241458528Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733241458528Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733241458528Cleaning up temporary data from old regions at 1733241458536 (+8 ms)Region opened successfully at 1733241458542 (+6 ms) 2024-12-03T15:57:38,542 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 20f1b3d193b989a09e21a98de0f761b7, disabling compactions & flushes 2024-12-03T15:57:38,543 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1733241458453.20f1b3d193b989a09e21a98de0f761b7. 2024-12-03T15:57:38,543 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1733241458453.20f1b3d193b989a09e21a98de0f761b7. 2024-12-03T15:57:38,543 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1733241458453.20f1b3d193b989a09e21a98de0f761b7. after waiting 0 ms 2024-12-03T15:57:38,543 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1733241458453.20f1b3d193b989a09e21a98de0f761b7. 2024-12-03T15:57:38,543 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1733241458453.20f1b3d193b989a09e21a98de0f761b7. 2024-12-03T15:57:38,544 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 20f1b3d193b989a09e21a98de0f761b7: Waiting for close lock at 1733241458542Disabling compacts and flushes for region at 1733241458542Disabling writes for close at 1733241458543 (+1 ms)Writing region close event to WAL at 1733241458543Closed at 1733241458543 2024-12-03T15:57:38,548 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /hbase/WALs/hregion-71148152/hregion-71148152.1733241458458 not finished, retry = 0 2024-12-03T15:57:38,550 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40575 is added to blk_1073741925_1104 (size=175) 2024-12-03T15:57:38,550 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44827 is added to blk_1073741925_1104 (size=175) 2024-12-03T15:57:38,550 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46585 is added to blk_1073741925_1104 (size=175) 2024-12-03T15:57:38,651 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-03T15:57:38,651 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-71148152:(num 1733241458458) 2024-12-03T15:57:38,651 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-03T15:57:38,655 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:45015/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733241458452, archiveDir=hdfs://localhost:45015/hbase/oldWALs, maxLogs=32 2024-12-03T15:57:38,671 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733241458452/wal.1733241458655, exclude list is [], retry=0 2024-12-03T15:57:38,674 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40575,DS-309fe9b7-1997-44cb-b0c9-1ce3c04a71de,DISK] 2024-12-03T15:57:38,675 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46585,DS-5c6771fd-75c7-4775-ad88-ae5bbe88de5c,DISK] 2024-12-03T15:57:38,675 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44827,DS-e007f1cc-fc1b-4368-9716-e11e4fe2f4eb,DISK] 2024-12-03T15:57:38,713 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733241458452/wal.1733241458655 2024-12-03T15:57:38,713 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44513:44513),(127.0.0.1/127.0.0.1:37355:37355),(127.0.0.1/127.0.0.1:46731:46731)] 2024-12-03T15:57:38,713 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 20f1b3d193b989a09e21a98de0f761b7, NAME => 'testReplayEditsWrittenViaHRegion,,1733241458453.20f1b3d193b989a09e21a98de0f761b7.', STARTKEY => '', ENDKEY => ''} 2024-12-03T15:57:38,714 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1733241458453.20f1b3d193b989a09e21a98de0f761b7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T15:57:38,714 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 20f1b3d193b989a09e21a98de0f761b7 2024-12-03T15:57:38,714 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 20f1b3d193b989a09e21a98de0f761b7 2024-12-03T15:57:38,715 INFO [StoreOpener-20f1b3d193b989a09e21a98de0f761b7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 20f1b3d193b989a09e21a98de0f761b7 2024-12-03T15:57:38,716 INFO [StoreOpener-20f1b3d193b989a09e21a98de0f761b7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 20f1b3d193b989a09e21a98de0f761b7 columnFamilyName a 2024-12-03T15:57:38,716 DEBUG [StoreOpener-20f1b3d193b989a09e21a98de0f761b7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T15:57:38,717 INFO [StoreOpener-20f1b3d193b989a09e21a98de0f761b7-1 {}] regionserver.HStore(327): Store=20f1b3d193b989a09e21a98de0f761b7/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T15:57:38,717 INFO [StoreOpener-20f1b3d193b989a09e21a98de0f761b7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 20f1b3d193b989a09e21a98de0f761b7 2024-12-03T15:57:38,717 INFO [StoreOpener-20f1b3d193b989a09e21a98de0f761b7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 20f1b3d193b989a09e21a98de0f761b7 columnFamilyName b 2024-12-03T15:57:38,717 DEBUG [StoreOpener-20f1b3d193b989a09e21a98de0f761b7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T15:57:38,718 INFO [StoreOpener-20f1b3d193b989a09e21a98de0f761b7-1 {}] regionserver.HStore(327): Store=20f1b3d193b989a09e21a98de0f761b7/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T15:57:38,718 INFO [StoreOpener-20f1b3d193b989a09e21a98de0f761b7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 20f1b3d193b989a09e21a98de0f761b7 2024-12-03T15:57:38,719 INFO [StoreOpener-20f1b3d193b989a09e21a98de0f761b7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 20f1b3d193b989a09e21a98de0f761b7 columnFamilyName c 2024-12-03T15:57:38,719 DEBUG [StoreOpener-20f1b3d193b989a09e21a98de0f761b7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T15:57:38,719 INFO [StoreOpener-20f1b3d193b989a09e21a98de0f761b7-1 {}] regionserver.HStore(327): Store=20f1b3d193b989a09e21a98de0f761b7/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T15:57:38,719 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 20f1b3d193b989a09e21a98de0f761b7 2024-12-03T15:57:38,720 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45015/hbase/data/default/testReplayEditsWrittenViaHRegion/20f1b3d193b989a09e21a98de0f761b7 2024-12-03T15:57:38,721 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45015/hbase/data/default/testReplayEditsWrittenViaHRegion/20f1b3d193b989a09e21a98de0f761b7 2024-12-03T15:57:38,721 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 20f1b3d193b989a09e21a98de0f761b7 2024-12-03T15:57:38,722 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 20f1b3d193b989a09e21a98de0f761b7 2024-12-03T15:57:38,722 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-03T15:57:38,724 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 20f1b3d193b989a09e21a98de0f761b7 2024-12-03T15:57:38,725 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 20f1b3d193b989a09e21a98de0f761b7; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71005111, jitterRate=0.05805860459804535}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-03T15:57:38,725 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 20f1b3d193b989a09e21a98de0f761b7: Writing region info on filesystem at 1733241458714Initializing all the Stores at 1733241458715 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733241458715Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733241458715Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733241458715Cleaning up temporary data from old regions at 1733241458722 (+7 ms)Region opened successfully at 1733241458725 (+3 ms) 2024-12-03T15:57:38,739 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 20f1b3d193b989a09e21a98de0f761b7 3/3 column families, dataSize=870 B heapSize=2.31 KB 2024-12-03T15:57:38,769 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45015/hbase/data/default/testReplayEditsWrittenViaHRegion/20f1b3d193b989a09e21a98de0f761b7/.tmp/a/1ad6be1f281e49578cd1ba5ff5387f00 is 91, key is testReplayEditsWrittenViaHRegion/a:x0/1733241458725/Put/seqid=0 2024-12-03T15:57:38,780 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46585 is added to blk_1073741928_1107 (size=5958) 2024-12-03T15:57:38,781 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44827 is added to blk_1073741928_1107 (size=5958) 2024-12-03T15:57:38,781 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40575 is added to blk_1073741928_1107 (size=5958) 2024-12-03T15:57:38,782 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:45015/hbase/data/default/testReplayEditsWrittenViaHRegion/20f1b3d193b989a09e21a98de0f761b7/.tmp/a/1ad6be1f281e49578cd1ba5ff5387f00 2024-12-03T15:57:38,790 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45015/hbase/data/default/testReplayEditsWrittenViaHRegion/20f1b3d193b989a09e21a98de0f761b7/.tmp/a/1ad6be1f281e49578cd1ba5ff5387f00 as hdfs://localhost:45015/hbase/data/default/testReplayEditsWrittenViaHRegion/20f1b3d193b989a09e21a98de0f761b7/a/1ad6be1f281e49578cd1ba5ff5387f00 2024-12-03T15:57:38,795 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45015/hbase/data/default/testReplayEditsWrittenViaHRegion/20f1b3d193b989a09e21a98de0f761b7/a/1ad6be1f281e49578cd1ba5ff5387f00, entries=10, sequenceid=13, filesize=5.8 K 2024-12-03T15:57:38,797 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~870 B/870, heapSize ~1.80 KB/1840, currentSize=0 B/0 for 20f1b3d193b989a09e21a98de0f761b7 in 57ms, sequenceid=13, compaction requested=false 2024-12-03T15:57:38,797 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 20f1b3d193b989a09e21a98de0f761b7: 2024-12-03T15:57:38,820 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 20f1b3d193b989a09e21a98de0f761b7, disabling compactions & flushes 2024-12-03T15:57:38,820 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1733241458453.20f1b3d193b989a09e21a98de0f761b7. 2024-12-03T15:57:38,821 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1733241458453.20f1b3d193b989a09e21a98de0f761b7. 2024-12-03T15:57:38,821 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1733241458453.20f1b3d193b989a09e21a98de0f761b7. after waiting 0 ms 2024-12-03T15:57:38,821 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1733241458453.20f1b3d193b989a09e21a98de0f761b7. 2024-12-03T15:57:38,822 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 1740 in region testReplayEditsWrittenViaHRegion,,1733241458453.20f1b3d193b989a09e21a98de0f761b7. 2024-12-03T15:57:38,822 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1733241458453.20f1b3d193b989a09e21a98de0f761b7. 2024-12-03T15:57:38,822 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 20f1b3d193b989a09e21a98de0f761b7: Waiting for close lock at 1733241458820Disabling compacts and flushes for region at 1733241458820Disabling writes for close at 1733241458821 (+1 ms)Writing region close event to WAL at 1733241458822 (+1 ms)Closed at 1733241458822 2024-12-03T15:57:38,828 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44827 is added to blk_1073741927_1106 (size=6243) 2024-12-03T15:57:38,828 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46585 is added to blk_1073741927_1106 (size=6243) 2024-12-03T15:57:38,829 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40575 is added to blk_1073741927_1106 (size=6243) 2024-12-03T15:57:38,846 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:45015/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733241458452/wal.1733241458655, size=6.1 K (6243bytes) 2024-12-03T15:57:38,846 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:45015/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733241458452/wal.1733241458655 2024-12-03T15:57:38,847 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:45015/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733241458452/wal.1733241458655 after 1ms 2024-12-03T15:57:38,880 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-03T15:57:38,880 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:45015/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733241458452/wal.1733241458655 took 34ms 2024-12-03T15:57:38,882 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:45015/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733241458452/wal.1733241458655 so closing down 2024-12-03T15:57:38,882 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-03T15:57:38,884 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1733241458655.temp 2024-12-03T15:57:38,915 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:45015/hbase/data/default/testReplayEditsWrittenViaHRegion/20f1b3d193b989a09e21a98de0f761b7/recovered.edits/0000000000000000003-wal.1733241458655.temp 2024-12-03T15:57:38,916 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-03T15:57:38,927 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44827 is added to blk_1073741929_1108 (size=5575) 2024-12-03T15:57:38,927 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46585 is added to blk_1073741929_1108 (size=5575) 2024-12-03T15:57:38,931 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:45015/hbase/data/default/testReplayEditsWrittenViaHRegion/20f1b3d193b989a09e21a98de0f761b7/recovered.edits/0000000000000000003-wal.1733241458655.temp (wrote 30 edits, skipped 0 edits in 0 ms) 2024-12-03T15:57:38,931 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40575 is added to blk_1073741929_1108 (size=5575) 2024-12-03T15:57:38,932 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:45015/hbase/data/default/testReplayEditsWrittenViaHRegion/20f1b3d193b989a09e21a98de0f761b7/recovered.edits/0000000000000000003-wal.1733241458655.temp to hdfs://localhost:45015/hbase/data/default/testReplayEditsWrittenViaHRegion/20f1b3d193b989a09e21a98de0f761b7/recovered.edits/0000000000000000035 2024-12-03T15:57:38,932 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 32 edits across 1 Regions in 52 ms; skipped=2; WAL=hdfs://localhost:45015/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733241458452/wal.1733241458655, size=6.1 K, length=6243, corrupted=false, cancelled=false 2024-12-03T15:57:38,933 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:45015/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733241458452/wal.1733241458655, journal: Splitting hdfs://localhost:45015/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733241458452/wal.1733241458655, size=6.1 K (6243bytes) at 1733241458846Finishing writing output for hdfs://localhost:45015/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733241458452/wal.1733241458655 so closing down at 1733241458882 (+36 ms)Creating recovered edits writer path=hdfs://localhost:45015/hbase/data/default/testReplayEditsWrittenViaHRegion/20f1b3d193b989a09e21a98de0f761b7/recovered.edits/0000000000000000003-wal.1733241458655.temp at 1733241458915 (+33 ms)3 split writer threads finished at 1733241458916 (+1 ms)Closed recovered edits writer path=hdfs://localhost:45015/hbase/data/default/testReplayEditsWrittenViaHRegion/20f1b3d193b989a09e21a98de0f761b7/recovered.edits/0000000000000000003-wal.1733241458655.temp (wrote 30 edits, skipped 0 edits in 0 ms) at 1733241458931 (+15 ms)Rename recovered edits hdfs://localhost:45015/hbase/data/default/testReplayEditsWrittenViaHRegion/20f1b3d193b989a09e21a98de0f761b7/recovered.edits/0000000000000000003-wal.1733241458655.temp to hdfs://localhost:45015/hbase/data/default/testReplayEditsWrittenViaHRegion/20f1b3d193b989a09e21a98de0f761b7/recovered.edits/0000000000000000035 at 1733241458932 (+1 ms)Processed 32 edits across 1 Regions in 52 ms; skipped=2; WAL=hdfs://localhost:45015/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733241458452/wal.1733241458655, size=6.1 K, length=6243, corrupted=false, cancelled=false at 1733241458932 2024-12-03T15:57:38,940 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:45015/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733241458452/wal.1733241458655 to hdfs://localhost:45015/hbase/oldWALs/wal.1733241458655 2024-12-03T15:57:38,941 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:45015/hbase/data/default/testReplayEditsWrittenViaHRegion/20f1b3d193b989a09e21a98de0f761b7/recovered.edits/0000000000000000035 2024-12-03T15:57:38,941 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-03T15:57:38,943 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:45015/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733241458452, archiveDir=hdfs://localhost:45015/hbase/oldWALs, maxLogs=32 2024-12-03T15:57:38,958 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733241458452/wal.1733241458944, exclude list is [], retry=0 2024-12-03T15:57:38,961 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44827,DS-e007f1cc-fc1b-4368-9716-e11e4fe2f4eb,DISK] 2024-12-03T15:57:38,961 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40575,DS-309fe9b7-1997-44cb-b0c9-1ce3c04a71de,DISK] 2024-12-03T15:57:38,961 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46585,DS-5c6771fd-75c7-4775-ad88-ae5bbe88de5c,DISK] 2024-12-03T15:57:39,001 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733241458452/wal.1733241458944 2024-12-03T15:57:39,001 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46731:46731),(127.0.0.1/127.0.0.1:44513:44513),(127.0.0.1/127.0.0.1:37355:37355)] 2024-12-03T15:57:39,001 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 20f1b3d193b989a09e21a98de0f761b7, NAME => 'testReplayEditsWrittenViaHRegion,,1733241458453.20f1b3d193b989a09e21a98de0f761b7.', STARTKEY => '', ENDKEY => ''} 2024-12-03T15:57:39,002 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1733241458453.20f1b3d193b989a09e21a98de0f761b7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T15:57:39,002 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 20f1b3d193b989a09e21a98de0f761b7 2024-12-03T15:57:39,002 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 20f1b3d193b989a09e21a98de0f761b7 2024-12-03T15:57:39,004 INFO [StoreOpener-20f1b3d193b989a09e21a98de0f761b7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 20f1b3d193b989a09e21a98de0f761b7 2024-12-03T15:57:39,005 INFO [StoreOpener-20f1b3d193b989a09e21a98de0f761b7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 20f1b3d193b989a09e21a98de0f761b7 columnFamilyName a 2024-12-03T15:57:39,005 DEBUG [StoreOpener-20f1b3d193b989a09e21a98de0f761b7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T15:57:39,012 DEBUG [StoreOpener-20f1b3d193b989a09e21a98de0f761b7-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:45015/hbase/data/default/testReplayEditsWrittenViaHRegion/20f1b3d193b989a09e21a98de0f761b7/a/1ad6be1f281e49578cd1ba5ff5387f00 2024-12-03T15:57:39,012 INFO [StoreOpener-20f1b3d193b989a09e21a98de0f761b7-1 {}] regionserver.HStore(327): Store=20f1b3d193b989a09e21a98de0f761b7/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T15:57:39,012 INFO [StoreOpener-20f1b3d193b989a09e21a98de0f761b7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 20f1b3d193b989a09e21a98de0f761b7 2024-12-03T15:57:39,014 INFO [StoreOpener-20f1b3d193b989a09e21a98de0f761b7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 20f1b3d193b989a09e21a98de0f761b7 columnFamilyName b 2024-12-03T15:57:39,014 DEBUG [StoreOpener-20f1b3d193b989a09e21a98de0f761b7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T15:57:39,014 INFO [StoreOpener-20f1b3d193b989a09e21a98de0f761b7-1 {}] regionserver.HStore(327): Store=20f1b3d193b989a09e21a98de0f761b7/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T15:57:39,014 INFO [StoreOpener-20f1b3d193b989a09e21a98de0f761b7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 20f1b3d193b989a09e21a98de0f761b7 2024-12-03T15:57:39,015 INFO [StoreOpener-20f1b3d193b989a09e21a98de0f761b7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 20f1b3d193b989a09e21a98de0f761b7 columnFamilyName c 2024-12-03T15:57:39,015 DEBUG [StoreOpener-20f1b3d193b989a09e21a98de0f761b7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T15:57:39,016 INFO [StoreOpener-20f1b3d193b989a09e21a98de0f761b7-1 {}] regionserver.HStore(327): Store=20f1b3d193b989a09e21a98de0f761b7/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T15:57:39,016 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 20f1b3d193b989a09e21a98de0f761b7 2024-12-03T15:57:39,017 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45015/hbase/data/default/testReplayEditsWrittenViaHRegion/20f1b3d193b989a09e21a98de0f761b7 2024-12-03T15:57:39,019 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:45015/hbase/data/default/testReplayEditsWrittenViaHRegion/20f1b3d193b989a09e21a98de0f761b7 2024-12-03T15:57:39,019 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:45015/hbase/data/default/testReplayEditsWrittenViaHRegion/20f1b3d193b989a09e21a98de0f761b7/recovered.edits/0000000000000000035 2024-12-03T15:57:39,062 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-03T15:57:39,064 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 20, skipped 10, firstSequenceIdInLog=3, maxSequenceIdInLog=35, path=hdfs://localhost:45015/hbase/data/default/testReplayEditsWrittenViaHRegion/20f1b3d193b989a09e21a98de0f761b7/recovered.edits/0000000000000000035 2024-12-03T15:57:39,064 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 20f1b3d193b989a09e21a98de0f761b7 3/3 column families, dataSize=1.70 KB heapSize=3.88 KB 2024-12-03T15:57:39,087 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45015/hbase/data/default/testReplayEditsWrittenViaHRegion/20f1b3d193b989a09e21a98de0f761b7/.tmp/b/52ec0c17a0ba4f56acc11d7b3c975db5 is 91, key is testReplayEditsWrittenViaHRegion/b:x0/1733241458797/Put/seqid=0 2024-12-03T15:57:39,095 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40575 is added to blk_1073741931_1110 (size=5958) 2024-12-03T15:57:39,096 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44827 is added to blk_1073741931_1110 (size=5958) 2024-12-03T15:57:39,097 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46585 is added to blk_1073741931_1110 (size=5958) 2024-12-03T15:57:39,097 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=35 (bloomFilter=true), to=hdfs://localhost:45015/hbase/data/default/testReplayEditsWrittenViaHRegion/20f1b3d193b989a09e21a98de0f761b7/.tmp/b/52ec0c17a0ba4f56acc11d7b3c975db5 2024-12-03T15:57:39,121 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45015/hbase/data/default/testReplayEditsWrittenViaHRegion/20f1b3d193b989a09e21a98de0f761b7/.tmp/c/879f36fbacfc43fb9151d99714afc336 is 91, key is testReplayEditsWrittenViaHRegion/c:x0/1733241458807/Put/seqid=0 2024-12-03T15:57:39,137 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46585 is added to blk_1073741932_1111 (size=5958) 2024-12-03T15:57:39,137 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44827 is added to blk_1073741932_1111 (size=5958) 2024-12-03T15:57:39,137 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40575 is added to blk_1073741932_1111 (size=5958) 2024-12-03T15:57:39,138 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=35 (bloomFilter=true), to=hdfs://localhost:45015/hbase/data/default/testReplayEditsWrittenViaHRegion/20f1b3d193b989a09e21a98de0f761b7/.tmp/c/879f36fbacfc43fb9151d99714afc336 2024-12-03T15:57:39,148 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45015/hbase/data/default/testReplayEditsWrittenViaHRegion/20f1b3d193b989a09e21a98de0f761b7/.tmp/b/52ec0c17a0ba4f56acc11d7b3c975db5 as hdfs://localhost:45015/hbase/data/default/testReplayEditsWrittenViaHRegion/20f1b3d193b989a09e21a98de0f761b7/b/52ec0c17a0ba4f56acc11d7b3c975db5 2024-12-03T15:57:39,158 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45015/hbase/data/default/testReplayEditsWrittenViaHRegion/20f1b3d193b989a09e21a98de0f761b7/b/52ec0c17a0ba4f56acc11d7b3c975db5, entries=10, sequenceid=35, filesize=5.8 K 2024-12-03T15:57:39,162 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45015/hbase/data/default/testReplayEditsWrittenViaHRegion/20f1b3d193b989a09e21a98de0f761b7/.tmp/c/879f36fbacfc43fb9151d99714afc336 as hdfs://localhost:45015/hbase/data/default/testReplayEditsWrittenViaHRegion/20f1b3d193b989a09e21a98de0f761b7/c/879f36fbacfc43fb9151d99714afc336 2024-12-03T15:57:39,170 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45015/hbase/data/default/testReplayEditsWrittenViaHRegion/20f1b3d193b989a09e21a98de0f761b7/c/879f36fbacfc43fb9151d99714afc336, entries=10, sequenceid=35, filesize=5.8 K 2024-12-03T15:57:39,170 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~1.70 KB/1740, heapSize ~3.59 KB/3680, currentSize=0 B/0 for 20f1b3d193b989a09e21a98de0f761b7 in 106ms, sequenceid=35, compaction requested=false; wal=null 2024-12-03T15:57:39,171 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:45015/hbase/data/default/testReplayEditsWrittenViaHRegion/20f1b3d193b989a09e21a98de0f761b7/recovered.edits/0000000000000000035 2024-12-03T15:57:39,174 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 20f1b3d193b989a09e21a98de0f761b7 2024-12-03T15:57:39,174 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 20f1b3d193b989a09e21a98de0f761b7 2024-12-03T15:57:39,174 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-03T15:57:39,185 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 20f1b3d193b989a09e21a98de0f761b7 2024-12-03T15:57:39,192 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45015/hbase/data/default/testReplayEditsWrittenViaHRegion/20f1b3d193b989a09e21a98de0f761b7/recovered.edits/35.seqid, newMaxSeqId=35, maxSeqId=1 2024-12-03T15:57:39,193 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 20f1b3d193b989a09e21a98de0f761b7; next sequenceid=36; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73052931, jitterRate=0.08857350051403046}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-03T15:57:39,193 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 20f1b3d193b989a09e21a98de0f761b7: Writing region info on filesystem at 1733241459002Initializing all the Stores at 1733241459003 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733241459003Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733241459003Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733241459003Obtaining lock to block concurrent updates at 1733241459064 (+61 ms)Preparing flush snapshotting stores in 20f1b3d193b989a09e21a98de0f761b7 at 1733241459064Finished memstore snapshotting testReplayEditsWrittenViaHRegion,,1733241458453.20f1b3d193b989a09e21a98de0f761b7., syncing WAL and waiting on mvcc, flushsize=dataSize=1740, getHeapSize=3920, getOffHeapSize=0, getCellsCount=20 at 1733241459065 (+1 ms)Flushing stores of testReplayEditsWrittenViaHRegion,,1733241458453.20f1b3d193b989a09e21a98de0f761b7. at 1733241459065Flushing 20f1b3d193b989a09e21a98de0f761b7/b: creating writer at 1733241459065Flushing 20f1b3d193b989a09e21a98de0f761b7/b: appending metadata at 1733241459085 (+20 ms)Flushing 20f1b3d193b989a09e21a98de0f761b7/b: closing flushed file at 1733241459085Flushing 20f1b3d193b989a09e21a98de0f761b7/c: creating writer at 1733241459104 (+19 ms)Flushing 20f1b3d193b989a09e21a98de0f761b7/c: appending metadata at 1733241459121 (+17 ms)Flushing 20f1b3d193b989a09e21a98de0f761b7/c: closing flushed file at 1733241459121Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@54a64916: reopening flushed file at 1733241459145 (+24 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@337aa046: reopening flushed file at 1733241459158 (+13 ms)Finished flush of dataSize ~1.70 KB/1740, heapSize ~3.59 KB/3680, currentSize=0 B/0 for 20f1b3d193b989a09e21a98de0f761b7 in 106ms, sequenceid=35, compaction requested=false; wal=null at 1733241459170 (+12 ms)Cleaning up temporary data from old regions at 1733241459174 (+4 ms)Region opened successfully at 1733241459193 (+19 ms) 2024-12-03T15:57:39,330 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:45015/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733241458452/wal.1733241458944, size=0 (0bytes) 2024-12-03T15:57:39,330 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:45015/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733241458452/wal.1733241458944 might be still open, length is 0 2024-12-03T15:57:39,331 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:45015/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733241458452/wal.1733241458944 2024-12-03T15:57:39,332 WARN [IPC Server handler 0 on default port 45015 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733241458452/wal.1733241458944 has not been closed. Lease recovery is in progress. RecoveryId = 1112 for block blk_1073741930_1109 2024-12-03T15:57:39,332 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:45015/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733241458452/wal.1733241458944 after 1ms 2024-12-03T15:57:41,010 INFO [master/713339a81dd9:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-12-03T15:57:41,010 INFO [master/713339a81dd9:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-12-03T15:57:41,064 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-365845159_22 at /127.0.0.1:35448 [Receiving block BP-1122350750-172.17.0.3-1733241413817:blk_1073741930_1109] {}] datanode.DataXceiver(331): 127.0.0.1:44827:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:35448 dst: /127.0.0.1:44827 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:44827 remote=/127.0.0.1:35448]. Total timeout mills is 60000, 58204 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T15:57:41,065 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-365845159_22 at /127.0.0.1:42068 [Receiving block BP-1122350750-172.17.0.3-1733241413817:blk_1073741930_1109] {}] datanode.DataXceiver(331): 127.0.0.1:40575:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:42068 dst: /127.0.0.1:40575 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T15:57:41,065 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-365845159_22 at /127.0.0.1:60960 [Receiving block BP-1122350750-172.17.0.3-1733241413817:blk_1073741930_1109] {}] datanode.DataXceiver(331): 127.0.0.1:46585:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:60960 dst: /127.0.0.1:46585 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T15:57:41,067 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44827 is added to blk_1073741930_1112 (size=5567) 2024-12-03T15:57:41,068 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40575 is added to blk_1073741930_1112 (size=5567) 2024-12-03T15:57:41,068 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46585 is added to blk_1073741930_1112 (size=5567) 2024-12-03T15:57:43,333 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:45015/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733241458452/wal.1733241458944 after 4002ms 2024-12-03T15:57:43,364 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-03T15:57:43,364 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:45015/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733241458452/wal.1733241458944 took 4034ms 2024-12-03T15:57:43,366 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:45015/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733241458452/wal.1733241458944; continuing. 2024-12-03T15:57:43,366 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:45015/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733241458452/wal.1733241458944 so closing down 2024-12-03T15:57:43,366 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-03T15:57:43,368 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000037-wal.1733241458944.temp 2024-12-03T15:57:43,397 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:45015/hbase/data/default/testReplayEditsWrittenViaHRegion/20f1b3d193b989a09e21a98de0f761b7/recovered.edits/0000000000000000037-wal.1733241458944.temp 2024-12-03T15:57:43,398 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-03T15:57:43,403 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46585 is added to blk_1073741933_1113 (size=5575) 2024-12-03T15:57:43,404 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44827 is added to blk_1073741933_1113 (size=5575) 2024-12-03T15:57:43,404 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40575 is added to blk_1073741933_1113 (size=5575) 2024-12-03T15:57:43,405 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:45015/hbase/data/default/testReplayEditsWrittenViaHRegion/20f1b3d193b989a09e21a98de0f761b7/recovered.edits/0000000000000000037-wal.1733241458944.temp (wrote 30 edits, skipped 0 edits in 0 ms) 2024-12-03T15:57:43,406 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:45015/hbase/data/default/testReplayEditsWrittenViaHRegion/20f1b3d193b989a09e21a98de0f761b7/recovered.edits/0000000000000000037-wal.1733241458944.temp to hdfs://localhost:45015/hbase/data/default/testReplayEditsWrittenViaHRegion/20f1b3d193b989a09e21a98de0f761b7/recovered.edits/0000000000000000066 2024-12-03T15:57:43,406 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 30 edits across 1 Regions in 42 ms; skipped=0; WAL=hdfs://localhost:45015/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733241458452/wal.1733241458944, size=0, length=0, corrupted=false, cancelled=false 2024-12-03T15:57:43,406 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:45015/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733241458452/wal.1733241458944, journal: Splitting hdfs://localhost:45015/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733241458452/wal.1733241458944, size=0 (0bytes) at 1733241459330Finishing writing output for hdfs://localhost:45015/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733241458452/wal.1733241458944 so closing down at 1733241463366 (+4036 ms)Creating recovered edits writer path=hdfs://localhost:45015/hbase/data/default/testReplayEditsWrittenViaHRegion/20f1b3d193b989a09e21a98de0f761b7/recovered.edits/0000000000000000037-wal.1733241458944.temp at 1733241463397 (+31 ms)3 split writer threads finished at 1733241463398 (+1 ms)Closed recovered edits writer path=hdfs://localhost:45015/hbase/data/default/testReplayEditsWrittenViaHRegion/20f1b3d193b989a09e21a98de0f761b7/recovered.edits/0000000000000000037-wal.1733241458944.temp (wrote 30 edits, skipped 0 edits in 0 ms) at 1733241463405 (+7 ms)Rename recovered edits hdfs://localhost:45015/hbase/data/default/testReplayEditsWrittenViaHRegion/20f1b3d193b989a09e21a98de0f761b7/recovered.edits/0000000000000000037-wal.1733241458944.temp to hdfs://localhost:45015/hbase/data/default/testReplayEditsWrittenViaHRegion/20f1b3d193b989a09e21a98de0f761b7/recovered.edits/0000000000000000066 at 1733241463406 (+1 ms)Processed 30 edits across 1 Regions in 42 ms; skipped=0; WAL=hdfs://localhost:45015/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733241458452/wal.1733241458944, size=0, length=0, corrupted=false, cancelled=false at 1733241463406 2024-12-03T15:57:43,408 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:45015/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733241458452/wal.1733241458944 to hdfs://localhost:45015/hbase/oldWALs/wal.1733241458944 2024-12-03T15:57:43,408 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:45015/hbase/data/default/testReplayEditsWrittenViaHRegion/20f1b3d193b989a09e21a98de0f761b7/recovered.edits/0000000000000000066 2024-12-03T15:57:43,409 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-03T15:57:43,410 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:45015/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733241458452, archiveDir=hdfs://localhost:45015/hbase/oldWALs, maxLogs=32 2024-12-03T15:57:43,425 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733241458452/wal.1733241463410, exclude list is [], retry=0 2024-12-03T15:57:43,427 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46585,DS-5c6771fd-75c7-4775-ad88-ae5bbe88de5c,DISK] 2024-12-03T15:57:43,428 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44827,DS-e007f1cc-fc1b-4368-9716-e11e4fe2f4eb,DISK] 2024-12-03T15:57:43,428 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40575,DS-309fe9b7-1997-44cb-b0c9-1ce3c04a71de,DISK] 2024-12-03T15:57:43,457 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733241458452/wal.1733241463410 2024-12-03T15:57:43,457 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37355:37355),(127.0.0.1/127.0.0.1:46731:46731),(127.0.0.1/127.0.0.1:44513:44513)] 2024-12-03T15:57:43,457 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1733241458453.20f1b3d193b989a09e21a98de0f761b7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T15:57:43,459 INFO [StoreOpener-20f1b3d193b989a09e21a98de0f761b7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 20f1b3d193b989a09e21a98de0f761b7 2024-12-03T15:57:43,460 INFO [StoreOpener-20f1b3d193b989a09e21a98de0f761b7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 20f1b3d193b989a09e21a98de0f761b7 columnFamilyName a 2024-12-03T15:57:43,460 DEBUG [StoreOpener-20f1b3d193b989a09e21a98de0f761b7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T15:57:43,465 DEBUG [StoreOpener-20f1b3d193b989a09e21a98de0f761b7-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:45015/hbase/data/default/testReplayEditsWrittenViaHRegion/20f1b3d193b989a09e21a98de0f761b7/a/1ad6be1f281e49578cd1ba5ff5387f00 2024-12-03T15:57:43,465 INFO [StoreOpener-20f1b3d193b989a09e21a98de0f761b7-1 {}] regionserver.HStore(327): Store=20f1b3d193b989a09e21a98de0f761b7/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T15:57:43,465 INFO [StoreOpener-20f1b3d193b989a09e21a98de0f761b7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 20f1b3d193b989a09e21a98de0f761b7 2024-12-03T15:57:43,466 INFO [StoreOpener-20f1b3d193b989a09e21a98de0f761b7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 20f1b3d193b989a09e21a98de0f761b7 columnFamilyName b 2024-12-03T15:57:43,466 DEBUG [StoreOpener-20f1b3d193b989a09e21a98de0f761b7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T15:57:43,470 DEBUG [StoreOpener-20f1b3d193b989a09e21a98de0f761b7-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:45015/hbase/data/default/testReplayEditsWrittenViaHRegion/20f1b3d193b989a09e21a98de0f761b7/b/52ec0c17a0ba4f56acc11d7b3c975db5 2024-12-03T15:57:43,471 INFO [StoreOpener-20f1b3d193b989a09e21a98de0f761b7-1 {}] regionserver.HStore(327): Store=20f1b3d193b989a09e21a98de0f761b7/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T15:57:43,471 INFO [StoreOpener-20f1b3d193b989a09e21a98de0f761b7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 20f1b3d193b989a09e21a98de0f761b7 2024-12-03T15:57:43,472 INFO [StoreOpener-20f1b3d193b989a09e21a98de0f761b7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 20f1b3d193b989a09e21a98de0f761b7 columnFamilyName c 2024-12-03T15:57:43,472 DEBUG [StoreOpener-20f1b3d193b989a09e21a98de0f761b7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T15:57:43,477 DEBUG [StoreOpener-20f1b3d193b989a09e21a98de0f761b7-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:45015/hbase/data/default/testReplayEditsWrittenViaHRegion/20f1b3d193b989a09e21a98de0f761b7/c/879f36fbacfc43fb9151d99714afc336 2024-12-03T15:57:43,478 INFO [StoreOpener-20f1b3d193b989a09e21a98de0f761b7-1 {}] regionserver.HStore(327): Store=20f1b3d193b989a09e21a98de0f761b7/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T15:57:43,478 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 20f1b3d193b989a09e21a98de0f761b7 2024-12-03T15:57:43,479 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45015/hbase/data/default/testReplayEditsWrittenViaHRegion/20f1b3d193b989a09e21a98de0f761b7 2024-12-03T15:57:43,480 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:45015/hbase/data/default/testReplayEditsWrittenViaHRegion/20f1b3d193b989a09e21a98de0f761b7 2024-12-03T15:57:43,481 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:45015/hbase/data/default/testReplayEditsWrittenViaHRegion/20f1b3d193b989a09e21a98de0f761b7/recovered.edits/0000000000000000066 2024-12-03T15:57:43,512 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-03T15:57:43,518 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 30, skipped 0, firstSequenceIdInLog=37, maxSequenceIdInLog=66, path=hdfs://localhost:45015/hbase/data/default/testReplayEditsWrittenViaHRegion/20f1b3d193b989a09e21a98de0f761b7/recovered.edits/0000000000000000066 2024-12-03T15:57:43,518 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 20f1b3d193b989a09e21a98de0f761b7 3/3 column families, dataSize=2.55 KB heapSize=5.44 KB 2024-12-03T15:57:43,535 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45015/hbase/data/default/testReplayEditsWrittenViaHRegion/20f1b3d193b989a09e21a98de0f761b7/.tmp/a/76c9fefe3c6b4ee4833a394d7c52d63c is 91, key is testReplayEditsWrittenViaHRegion/a:y0/1733241459209/Put/seqid=0 2024-12-03T15:57:43,542 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44827 is added to blk_1073741935_1115 (size=5958) 2024-12-03T15:57:43,542 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40575 is added to blk_1073741935_1115 (size=5958) 2024-12-03T15:57:43,543 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46585 is added to blk_1073741935_1115 (size=5958) 2024-12-03T15:57:43,547 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:45015/hbase/data/default/testReplayEditsWrittenViaHRegion/20f1b3d193b989a09e21a98de0f761b7/.tmp/a/76c9fefe3c6b4ee4833a394d7c52d63c 2024-12-03T15:57:43,568 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45015/hbase/data/default/testReplayEditsWrittenViaHRegion/20f1b3d193b989a09e21a98de0f761b7/.tmp/b/51b41ac3d80f4891bde1132966321f0b is 91, key is testReplayEditsWrittenViaHRegion/b:y0/1733241459240/Put/seqid=0 2024-12-03T15:57:43,574 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46585 is added to blk_1073741936_1116 (size=5958) 2024-12-03T15:57:43,575 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44827 is added to blk_1073741936_1116 (size=5958) 2024-12-03T15:57:43,575 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40575 is added to blk_1073741936_1116 (size=5958) 2024-12-03T15:57:43,576 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:45015/hbase/data/default/testReplayEditsWrittenViaHRegion/20f1b3d193b989a09e21a98de0f761b7/.tmp/b/51b41ac3d80f4891bde1132966321f0b 2024-12-03T15:57:43,596 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45015/hbase/data/default/testReplayEditsWrittenViaHRegion/20f1b3d193b989a09e21a98de0f761b7/.tmp/c/8f73da3329c94971bcc9f416d5b17ad1 is 91, key is testReplayEditsWrittenViaHRegion/c:y0/1733241459254/Put/seqid=0 2024-12-03T15:57:43,602 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46585 is added to blk_1073741937_1117 (size=5958) 2024-12-03T15:57:43,603 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44827 is added to blk_1073741937_1117 (size=5958) 2024-12-03T15:57:43,603 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40575 is added to blk_1073741937_1117 (size=5958) 2024-12-03T15:57:43,604 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:45015/hbase/data/default/testReplayEditsWrittenViaHRegion/20f1b3d193b989a09e21a98de0f761b7/.tmp/c/8f73da3329c94971bcc9f416d5b17ad1 2024-12-03T15:57:43,609 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45015/hbase/data/default/testReplayEditsWrittenViaHRegion/20f1b3d193b989a09e21a98de0f761b7/.tmp/a/76c9fefe3c6b4ee4833a394d7c52d63c as hdfs://localhost:45015/hbase/data/default/testReplayEditsWrittenViaHRegion/20f1b3d193b989a09e21a98de0f761b7/a/76c9fefe3c6b4ee4833a394d7c52d63c 2024-12-03T15:57:43,614 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45015/hbase/data/default/testReplayEditsWrittenViaHRegion/20f1b3d193b989a09e21a98de0f761b7/a/76c9fefe3c6b4ee4833a394d7c52d63c, entries=10, sequenceid=66, filesize=5.8 K 2024-12-03T15:57:43,615 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45015/hbase/data/default/testReplayEditsWrittenViaHRegion/20f1b3d193b989a09e21a98de0f761b7/.tmp/b/51b41ac3d80f4891bde1132966321f0b as hdfs://localhost:45015/hbase/data/default/testReplayEditsWrittenViaHRegion/20f1b3d193b989a09e21a98de0f761b7/b/51b41ac3d80f4891bde1132966321f0b 2024-12-03T15:57:43,620 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45015/hbase/data/default/testReplayEditsWrittenViaHRegion/20f1b3d193b989a09e21a98de0f761b7/b/51b41ac3d80f4891bde1132966321f0b, entries=10, sequenceid=66, filesize=5.8 K 2024-12-03T15:57:43,621 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45015/hbase/data/default/testReplayEditsWrittenViaHRegion/20f1b3d193b989a09e21a98de0f761b7/.tmp/c/8f73da3329c94971bcc9f416d5b17ad1 as hdfs://localhost:45015/hbase/data/default/testReplayEditsWrittenViaHRegion/20f1b3d193b989a09e21a98de0f761b7/c/8f73da3329c94971bcc9f416d5b17ad1 2024-12-03T15:57:43,625 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45015/hbase/data/default/testReplayEditsWrittenViaHRegion/20f1b3d193b989a09e21a98de0f761b7/c/8f73da3329c94971bcc9f416d5b17ad1, entries=10, sequenceid=66, filesize=5.8 K 2024-12-03T15:57:43,625 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~2.55 KB/2610, heapSize ~5.39 KB/5520, currentSize=0 B/0 for 20f1b3d193b989a09e21a98de0f761b7 in 107ms, sequenceid=66, compaction requested=false; wal=null 2024-12-03T15:57:43,626 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:45015/hbase/data/default/testReplayEditsWrittenViaHRegion/20f1b3d193b989a09e21a98de0f761b7/recovered.edits/0000000000000000066 2024-12-03T15:57:43,627 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 20f1b3d193b989a09e21a98de0f761b7 2024-12-03T15:57:43,627 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 20f1b3d193b989a09e21a98de0f761b7 2024-12-03T15:57:43,628 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-03T15:57:43,629 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 20f1b3d193b989a09e21a98de0f761b7 2024-12-03T15:57:43,631 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45015/hbase/data/default/testReplayEditsWrittenViaHRegion/20f1b3d193b989a09e21a98de0f761b7/recovered.edits/66.seqid, newMaxSeqId=66, maxSeqId=35 2024-12-03T15:57:43,632 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 20f1b3d193b989a09e21a98de0f761b7; next sequenceid=67; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60048018, jitterRate=-0.10521480441093445}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-03T15:57:43,632 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 20f1b3d193b989a09e21a98de0f761b7: Writing region info on filesystem at 1733241463457Initializing all the Stores at 1733241463458 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733241463458Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733241463459 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733241463459Obtaining lock to block concurrent updates at 1733241463518 (+59 ms)Preparing flush snapshotting stores in 20f1b3d193b989a09e21a98de0f761b7 at 1733241463518Finished memstore snapshotting testReplayEditsWrittenViaHRegion,,1733241458453.20f1b3d193b989a09e21a98de0f761b7., syncing WAL and waiting on mvcc, flushsize=dataSize=2610, getHeapSize=5520, getOffHeapSize=0, getCellsCount=30 at 1733241463518Flushing stores of testReplayEditsWrittenViaHRegion,,1733241458453.20f1b3d193b989a09e21a98de0f761b7. at 1733241463518Flushing 20f1b3d193b989a09e21a98de0f761b7/a: creating writer at 1733241463518Flushing 20f1b3d193b989a09e21a98de0f761b7/a: appending metadata at 1733241463535 (+17 ms)Flushing 20f1b3d193b989a09e21a98de0f761b7/a: closing flushed file at 1733241463535Flushing 20f1b3d193b989a09e21a98de0f761b7/b: creating writer at 1733241463551 (+16 ms)Flushing 20f1b3d193b989a09e21a98de0f761b7/b: appending metadata at 1733241463567 (+16 ms)Flushing 20f1b3d193b989a09e21a98de0f761b7/b: closing flushed file at 1733241463567Flushing 20f1b3d193b989a09e21a98de0f761b7/c: creating writer at 1733241463581 (+14 ms)Flushing 20f1b3d193b989a09e21a98de0f761b7/c: appending metadata at 1733241463596 (+15 ms)Flushing 20f1b3d193b989a09e21a98de0f761b7/c: closing flushed file at 1733241463596Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1806537b: reopening flushed file at 1733241463608 (+12 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@55496df5: reopening flushed file at 1733241463614 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@774564a6: reopening flushed file at 1733241463620 (+6 ms)Finished flush of dataSize ~2.55 KB/2610, heapSize ~5.39 KB/5520, currentSize=0 B/0 for 20f1b3d193b989a09e21a98de0f761b7 in 107ms, sequenceid=66, compaction requested=false; wal=null at 1733241463625 (+5 ms)Cleaning up temporary data from old regions at 1733241463627 (+2 ms)Region opened successfully at 1733241463632 (+5 ms) 2024-12-03T15:57:43,646 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 20f1b3d193b989a09e21a98de0f761b7, disabling compactions & flushes 2024-12-03T15:57:43,646 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1733241458453.20f1b3d193b989a09e21a98de0f761b7. 2024-12-03T15:57:43,646 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1733241458453.20f1b3d193b989a09e21a98de0f761b7. 2024-12-03T15:57:43,646 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1733241458453.20f1b3d193b989a09e21a98de0f761b7. after waiting 0 ms 2024-12-03T15:57:43,646 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1733241458453.20f1b3d193b989a09e21a98de0f761b7. 2024-12-03T15:57:43,651 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1733241458453.20f1b3d193b989a09e21a98de0f761b7. 2024-12-03T15:57:43,651 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 20f1b3d193b989a09e21a98de0f761b7: Waiting for close lock at 1733241463646Disabling compacts and flushes for region at 1733241463646Disabling writes for close at 1733241463646Writing region close event to WAL at 1733241463651 (+5 ms)Closed at 1733241463651 2024-12-03T15:57:43,654 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44827 is added to blk_1073741934_1114 (size=175) 2024-12-03T15:57:43,654 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46585 is added to blk_1073741934_1114 (size=175) 2024-12-03T15:57:43,654 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40575 is added to blk_1073741934_1114 (size=175) 2024-12-03T15:57:43,656 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-03T15:57:43,656 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1733241463410) 2024-12-03T15:57:43,674 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsWrittenViaHRegion Thread=445 (was 440) Potentially hanging thread: AsyncFSWAL-28-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_637810209_22 at /127.0.0.1:44606 [Waiting for operation #13] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkinstestReplayEditsWrittenViaHRegion@localhost:45015 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1616464420) connection to localhost/127.0.0.1:45015 from jenkinstestReplayEditsWrittenViaHRegion java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_637810209_22 at /127.0.0.1:39114 [Waiting for operation #12] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-28-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_637810209_22 at /127.0.0.1:33508 [Waiting for operation #18] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-28-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1410 (was 1348) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=726 (was 746), ProcessCount=11 (was 11), AvailableMemoryMB=4898 (was 4895) - AvailableMemoryMB LEAK? - 2024-12-03T15:57:43,674 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1410 is superior to 1024 2024-12-03T15:57:43,674 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-12-03T15:57:43,675 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-03T15:57:43,675 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-03T15:57:43,675 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T15:57:43,675 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T15:57:43,675 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T15:57:43,676 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-12-03T15:57:43,676 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=954470236, stopped=false 2024-12-03T15:57:43,676 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=713339a81dd9,34663,1733241417250 2024-12-03T15:57:43,678 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42147-0x1017926911e0001, quorum=127.0.0.1:61024, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-03T15:57:43,678 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34663-0x1017926911e0000, quorum=127.0.0.1:61024, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-03T15:57:43,678 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34663-0x1017926911e0000, quorum=127.0.0.1:61024, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T15:57:43,678 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42147-0x1017926911e0001, quorum=127.0.0.1:61024, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T15:57:43,678 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34819-0x1017926911e0002, quorum=127.0.0.1:61024, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-03T15:57:43,678 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-03T15:57:43,678 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34819-0x1017926911e0002, quorum=127.0.0.1:61024, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T15:57:43,679 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:34663-0x1017926911e0000, quorum=127.0.0.1:61024, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-03T15:57:43,679 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-03T15:57:43,679 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-03T15:57:43,679 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T15:57:43,680 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '713339a81dd9,42147,1733241418180' ***** 2024-12-03T15:57:43,680 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-03T15:57:43,680 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '713339a81dd9,34819,1733241418314' ***** 2024-12-03T15:57:43,680 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-03T15:57:43,680 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:42147-0x1017926911e0001, quorum=127.0.0.1:61024, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-03T15:57:43,680 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:34819-0x1017926911e0002, quorum=127.0.0.1:61024, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-03T15:57:43,680 INFO [RS:1;713339a81dd9:34819 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-03T15:57:43,680 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-03T15:57:43,680 INFO [RS:1;713339a81dd9:34819 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-03T15:57:43,681 INFO [RS:1;713339a81dd9:34819 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-03T15:57:43,681 INFO [RS:1;713339a81dd9:34819 {}] regionserver.HRegionServer(959): stopping server 713339a81dd9,34819,1733241418314 2024-12-03T15:57:43,681 INFO [RS:1;713339a81dd9:34819 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-03T15:57:43,681 INFO [RS:1;713339a81dd9:34819 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;713339a81dd9:34819. 2024-12-03T15:57:43,681 DEBUG [RS:1;713339a81dd9:34819 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-03T15:57:43,681 DEBUG [RS:1;713339a81dd9:34819 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T15:57:43,681 INFO [RS:1;713339a81dd9:34819 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-03T15:57:43,681 INFO [RS:1;713339a81dd9:34819 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-03T15:57:43,681 INFO [RS:1;713339a81dd9:34819 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-03T15:57:43,681 INFO [RS:1;713339a81dd9:34819 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-12-03T15:57:43,681 INFO [RS:1;713339a81dd9:34819 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-12-03T15:57:43,682 DEBUG [RS:1;713339a81dd9:34819 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-12-03T15:57:43,682 DEBUG [RS:1;713339a81dd9:34819 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-03T15:57:43,682 DEBUG [RS_CLOSE_META-regionserver/713339a81dd9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-03T15:57:43,682 INFO [RS_CLOSE_META-regionserver/713339a81dd9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-03T15:57:43,682 DEBUG [RS_CLOSE_META-regionserver/713339a81dd9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-03T15:57:43,682 DEBUG [RS_CLOSE_META-regionserver/713339a81dd9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-03T15:57:43,682 DEBUG [RS_CLOSE_META-regionserver/713339a81dd9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-03T15:57:43,682 INFO [RS_CLOSE_META-regionserver/713339a81dd9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=6.86 KB heapSize=11.45 KB 2024-12-03T15:57:43,683 INFO [RS:0;713339a81dd9:42147 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-03T15:57:43,683 INFO [RS:0;713339a81dd9:42147 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-03T15:57:43,683 INFO [RS:0;713339a81dd9:42147 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-03T15:57:43,683 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-03T15:57:43,683 INFO [RS:0;713339a81dd9:42147 {}] regionserver.HRegionServer(3091): Received CLOSE for 03c5565060bb604d3f080f88f20309db 2024-12-03T15:57:43,683 INFO [RS:0;713339a81dd9:42147 {}] regionserver.HRegionServer(959): stopping server 713339a81dd9,42147,1733241418180 2024-12-03T15:57:43,684 DEBUG [RS_CLOSE_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 03c5565060bb604d3f080f88f20309db, disabling compactions & flushes 2024-12-03T15:57:43,684 INFO [RS_CLOSE_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1733241437409.03c5565060bb604d3f080f88f20309db. 2024-12-03T15:57:43,684 DEBUG [RS_CLOSE_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733241437409.03c5565060bb604d3f080f88f20309db. 2024-12-03T15:57:43,684 DEBUG [RS_CLOSE_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733241437409.03c5565060bb604d3f080f88f20309db. after waiting 0 ms 2024-12-03T15:57:43,684 DEBUG [RS_CLOSE_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1733241437409.03c5565060bb604d3f080f88f20309db. 2024-12-03T15:57:43,684 INFO [RS:0;713339a81dd9:42147 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-03T15:57:43,685 INFO [RS:0;713339a81dd9:42147 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;713339a81dd9:42147. 2024-12-03T15:57:43,685 DEBUG [RS:0;713339a81dd9:42147 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-03T15:57:43,685 DEBUG [RS:0;713339a81dd9:42147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T15:57:43,685 INFO [RS:0;713339a81dd9:42147 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-12-03T15:57:43,685 DEBUG [RS:0;713339a81dd9:42147 {}] regionserver.HRegionServer(1325): Online Regions={03c5565060bb604d3f080f88f20309db=testReplayEditsAfterRegionMovedWithMultiCF,,1733241437409.03c5565060bb604d3f080f88f20309db.} 2024-12-03T15:57:43,685 DEBUG [RS:0;713339a81dd9:42147 {}] regionserver.HRegionServer(1351): Waiting on 03c5565060bb604d3f080f88f20309db 2024-12-03T15:57:43,691 DEBUG [RS_CLOSE_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45015/user/jenkins/test-data/c81ce3c5-e5b8-1c2f-869e-120d734cf311/data/default/testReplayEditsAfterRegionMovedWithMultiCF/03c5565060bb604d3f080f88f20309db/recovered.edits/20.seqid, newMaxSeqId=20, maxSeqId=17 2024-12-03T15:57:43,692 INFO [RS_CLOSE_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733241437409.03c5565060bb604d3f080f88f20309db. 2024-12-03T15:57:43,692 DEBUG [RS_CLOSE_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 03c5565060bb604d3f080f88f20309db: Waiting for close lock at 1733241463683Running coprocessor pre-close hooks at 1733241463684 (+1 ms)Disabling compacts and flushes for region at 1733241463684Disabling writes for close at 1733241463684Writing region close event to WAL at 1733241463685 (+1 ms)Running coprocessor post-close hooks at 1733241463692 (+7 ms)Closed at 1733241463692 2024-12-03T15:57:43,692 DEBUG [RS_CLOSE_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733241437409.03c5565060bb604d3f080f88f20309db. 2024-12-03T15:57:43,708 DEBUG [RS_CLOSE_META-regionserver/713339a81dd9:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45015/user/jenkins/test-data/c81ce3c5-e5b8-1c2f-869e-120d734cf311/data/hbase/meta/1588230740/.tmp/info/26d6906b30874092b48b838885639e19 is 205, key is testReplayEditsAfterRegionMovedWithMultiCF,,1733241437409.03c5565060bb604d3f080f88f20309db./info:regioninfo/1733241441148/Put/seqid=0 2024-12-03T15:57:43,721 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46585 is added to blk_1073741938_1118 (size=8243) 2024-12-03T15:57:43,721 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40575 is added to blk_1073741938_1118 (size=8243) 2024-12-03T15:57:43,721 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44827 is added to blk_1073741938_1118 (size=8243) 2024-12-03T15:57:43,722 INFO [RS_CLOSE_META-regionserver/713339a81dd9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.65 KB at sequenceid=21 (bloomFilter=true), to=hdfs://localhost:45015/user/jenkins/test-data/c81ce3c5-e5b8-1c2f-869e-120d734cf311/data/hbase/meta/1588230740/.tmp/info/26d6906b30874092b48b838885639e19 2024-12-03T15:57:43,744 DEBUG [RS_CLOSE_META-regionserver/713339a81dd9:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45015/user/jenkins/test-data/c81ce3c5-e5b8-1c2f-869e-120d734cf311/data/hbase/meta/1588230740/.tmp/ns/51ffc45fc5cd4cc09d1e7f548003c88f is 43, key is default/ns:d/1733241420783/Put/seqid=0 2024-12-03T15:57:43,748 INFO [regionserver/713339a81dd9:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-03T15:57:43,750 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46585 is added to blk_1073741939_1119 (size=5153) 2024-12-03T15:57:43,751 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40575 is added to blk_1073741939_1119 (size=5153) 2024-12-03T15:57:43,752 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44827 is added to blk_1073741939_1119 (size=5153) 2024-12-03T15:57:43,752 INFO [RS_CLOSE_META-regionserver/713339a81dd9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=21 (bloomFilter=true), to=hdfs://localhost:45015/user/jenkins/test-data/c81ce3c5-e5b8-1c2f-869e-120d734cf311/data/hbase/meta/1588230740/.tmp/ns/51ffc45fc5cd4cc09d1e7f548003c88f 2024-12-03T15:57:43,757 INFO [regionserver/713339a81dd9:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-03T15:57:43,775 DEBUG [RS_CLOSE_META-regionserver/713339a81dd9:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45015/user/jenkins/test-data/c81ce3c5-e5b8-1c2f-869e-120d734cf311/data/hbase/meta/1588230740/.tmp/table/dab3a549ab444a41a11169b6fcf6b326 is 78, key is testReplayEditsAfterRegionMovedWithMultiCF/table:state/1733241437886/Put/seqid=0 2024-12-03T15:57:43,781 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40575 is added to blk_1073741940_1120 (size=5431) 2024-12-03T15:57:43,781 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46585 is added to blk_1073741940_1120 (size=5431) 2024-12-03T15:57:43,782 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44827 is added to blk_1073741940_1120 (size=5431) 2024-12-03T15:57:43,782 INFO [RS_CLOSE_META-regionserver/713339a81dd9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=148 B at sequenceid=21 (bloomFilter=true), to=hdfs://localhost:45015/user/jenkins/test-data/c81ce3c5-e5b8-1c2f-869e-120d734cf311/data/hbase/meta/1588230740/.tmp/table/dab3a549ab444a41a11169b6fcf6b326 2024-12-03T15:57:43,789 DEBUG [RS_CLOSE_META-regionserver/713339a81dd9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45015/user/jenkins/test-data/c81ce3c5-e5b8-1c2f-869e-120d734cf311/data/hbase/meta/1588230740/.tmp/info/26d6906b30874092b48b838885639e19 as hdfs://localhost:45015/user/jenkins/test-data/c81ce3c5-e5b8-1c2f-869e-120d734cf311/data/hbase/meta/1588230740/info/26d6906b30874092b48b838885639e19 2024-12-03T15:57:43,795 INFO [RS_CLOSE_META-regionserver/713339a81dd9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45015/user/jenkins/test-data/c81ce3c5-e5b8-1c2f-869e-120d734cf311/data/hbase/meta/1588230740/info/26d6906b30874092b48b838885639e19, entries=18, sequenceid=21, filesize=8.0 K 2024-12-03T15:57:43,796 DEBUG [RS_CLOSE_META-regionserver/713339a81dd9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45015/user/jenkins/test-data/c81ce3c5-e5b8-1c2f-869e-120d734cf311/data/hbase/meta/1588230740/.tmp/ns/51ffc45fc5cd4cc09d1e7f548003c88f as hdfs://localhost:45015/user/jenkins/test-data/c81ce3c5-e5b8-1c2f-869e-120d734cf311/data/hbase/meta/1588230740/ns/51ffc45fc5cd4cc09d1e7f548003c88f 2024-12-03T15:57:43,801 INFO [RS_CLOSE_META-regionserver/713339a81dd9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45015/user/jenkins/test-data/c81ce3c5-e5b8-1c2f-869e-120d734cf311/data/hbase/meta/1588230740/ns/51ffc45fc5cd4cc09d1e7f548003c88f, entries=2, sequenceid=21, filesize=5.0 K 2024-12-03T15:57:43,802 DEBUG [RS_CLOSE_META-regionserver/713339a81dd9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45015/user/jenkins/test-data/c81ce3c5-e5b8-1c2f-869e-120d734cf311/data/hbase/meta/1588230740/.tmp/table/dab3a549ab444a41a11169b6fcf6b326 as hdfs://localhost:45015/user/jenkins/test-data/c81ce3c5-e5b8-1c2f-869e-120d734cf311/data/hbase/meta/1588230740/table/dab3a549ab444a41a11169b6fcf6b326 2024-12-03T15:57:43,807 INFO [RS_CLOSE_META-regionserver/713339a81dd9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45015/user/jenkins/test-data/c81ce3c5-e5b8-1c2f-869e-120d734cf311/data/hbase/meta/1588230740/table/dab3a549ab444a41a11169b6fcf6b326, entries=2, sequenceid=21, filesize=5.3 K 2024-12-03T15:57:43,808 INFO [RS_CLOSE_META-regionserver/713339a81dd9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~6.86 KB/7029, heapSize ~11.16 KB/11424, currentSize=0 B/0 for 1588230740 in 126ms, sequenceid=21, compaction requested=false 2024-12-03T15:57:43,814 DEBUG [RS_CLOSE_META-regionserver/713339a81dd9:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45015/user/jenkins/test-data/c81ce3c5-e5b8-1c2f-869e-120d734cf311/data/hbase/meta/1588230740/recovered.edits/24.seqid, newMaxSeqId=24, maxSeqId=1 2024-12-03T15:57:43,814 DEBUG [RS_CLOSE_META-regionserver/713339a81dd9:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-03T15:57:43,815 INFO [RS_CLOSE_META-regionserver/713339a81dd9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-03T15:57:43,815 DEBUG [RS_CLOSE_META-regionserver/713339a81dd9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733241463682Running coprocessor pre-close hooks at 1733241463682Disabling compacts and flushes for region at 1733241463682Disabling writes for close at 1733241463682Obtaining lock to block concurrent updates at 1733241463682Preparing flush snapshotting stores in 1588230740 at 1733241463682Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=7029, getHeapSize=11664, getOffHeapSize=0, getCellsCount=48 at 1733241463683 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1733241463683Flushing 1588230740/info: creating writer at 1733241463684 (+1 ms)Flushing 1588230740/info: appending metadata at 1733241463707 (+23 ms)Flushing 1588230740/info: closing flushed file at 1733241463707Flushing 1588230740/ns: creating writer at 1733241463728 (+21 ms)Flushing 1588230740/ns: appending metadata at 1733241463743 (+15 ms)Flushing 1588230740/ns: closing flushed file at 1733241463743Flushing 1588230740/table: creating writer at 1733241463758 (+15 ms)Flushing 1588230740/table: appending metadata at 1733241463774 (+16 ms)Flushing 1588230740/table: closing flushed file at 1733241463774Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@68526076: reopening flushed file at 1733241463788 (+14 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@522486a7: reopening flushed file at 1733241463795 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@95125ab: reopening flushed file at 1733241463801 (+6 ms)Finished flush of dataSize ~6.86 KB/7029, heapSize ~11.16 KB/11424, currentSize=0 B/0 for 1588230740 in 126ms, sequenceid=21, compaction requested=false at 1733241463809 (+8 ms)Writing region close event to WAL at 1733241463810 (+1 ms)Running coprocessor post-close hooks at 1733241463814 (+4 ms)Closed at 1733241463815 (+1 ms) 2024-12-03T15:57:43,815 DEBUG [RS_CLOSE_META-regionserver/713339a81dd9:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-12-03T15:57:43,882 INFO [RS:1;713339a81dd9:34819 {}] regionserver.HRegionServer(976): stopping server 713339a81dd9,34819,1733241418314; all regions closed. 2024-12-03T15:57:43,885 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46585 is added to blk_1073741836_1012 (size=9487) 2024-12-03T15:57:43,885 INFO [RS:0;713339a81dd9:42147 {}] regionserver.HRegionServer(976): stopping server 713339a81dd9,42147,1733241418180; all regions closed. 2024-12-03T15:57:43,885 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44827 is added to blk_1073741836_1012 (size=9487) 2024-12-03T15:57:43,885 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40575 is added to blk_1073741836_1012 (size=9487) 2024-12-03T15:57:43,888 DEBUG [RS:1;713339a81dd9:34819 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/c81ce3c5-e5b8-1c2f-869e-120d734cf311/oldWALs 2024-12-03T15:57:43,888 INFO [RS:1;713339a81dd9:34819 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 713339a81dd9%2C34819%2C1733241418314.meta:.meta(num 1733241420549) 2024-12-03T15:57:43,888 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44827 is added to blk_1073741834_1010 (size=3621) 2024-12-03T15:57:43,888 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46585 is added to blk_1073741834_1010 (size=3621) 2024-12-03T15:57:43,889 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40575 is added to blk_1073741834_1010 (size=3621) 2024-12-03T15:57:43,890 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40575 is added to blk_1073741835_1011 (size=175) 2024-12-03T15:57:43,890 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44827 is added to blk_1073741835_1011 (size=175) 2024-12-03T15:57:43,890 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46585 is added to blk_1073741835_1011 (size=175) 2024-12-03T15:57:43,892 DEBUG [RS:0;713339a81dd9:42147 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/c81ce3c5-e5b8-1c2f-869e-120d734cf311/oldWALs 2024-12-03T15:57:43,892 INFO [RS:0;713339a81dd9:42147 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 713339a81dd9%2C42147%2C1733241418180:(num 1733241420132) 2024-12-03T15:57:43,892 DEBUG [RS:0;713339a81dd9:42147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T15:57:43,892 INFO [RS:0;713339a81dd9:42147 {}] regionserver.LeaseManager(133): Closed leases 2024-12-03T15:57:43,892 INFO [RS:0;713339a81dd9:42147 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-03T15:57:43,893 DEBUG [RS:1;713339a81dd9:34819 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/c81ce3c5-e5b8-1c2f-869e-120d734cf311/oldWALs 2024-12-03T15:57:43,893 INFO [RS:1;713339a81dd9:34819 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 713339a81dd9%2C34819%2C1733241418314:(num 1733241420140) 2024-12-03T15:57:43,893 INFO [RS:0;713339a81dd9:42147 {}] hbase.ChoreService(370): Chore service for: regionserver/713339a81dd9:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-12-03T15:57:43,893 DEBUG [RS:1;713339a81dd9:34819 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T15:57:43,893 INFO [RS:1;713339a81dd9:34819 {}] regionserver.LeaseManager(133): Closed leases 2024-12-03T15:57:43,893 INFO [RS:1;713339a81dd9:34819 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-03T15:57:43,893 INFO [RS:0;713339a81dd9:42147 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-03T15:57:43,893 INFO [regionserver/713339a81dd9:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-03T15:57:43,893 INFO [RS:0;713339a81dd9:42147 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-03T15:57:43,893 INFO [RS:0;713339a81dd9:42147 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-03T15:57:43,893 INFO [RS:0;713339a81dd9:42147 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-03T15:57:43,893 INFO [RS:1;713339a81dd9:34819 {}] hbase.ChoreService(370): Chore service for: regionserver/713339a81dd9:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-12-03T15:57:43,893 INFO [RS:0;713339a81dd9:42147 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:42147 2024-12-03T15:57:43,893 INFO [RS:1;713339a81dd9:34819 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-03T15:57:43,893 INFO [regionserver/713339a81dd9:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-03T15:57:43,893 INFO [RS:1;713339a81dd9:34819 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:34819 2024-12-03T15:57:43,896 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34663-0x1017926911e0000, quorum=127.0.0.1:61024, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-03T15:57:43,896 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34819-0x1017926911e0002, quorum=127.0.0.1:61024, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/713339a81dd9,34819,1733241418314 2024-12-03T15:57:43,896 INFO [RS:1;713339a81dd9:34819 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-03T15:57:43,897 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42147-0x1017926911e0001, quorum=127.0.0.1:61024, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/713339a81dd9,42147,1733241418180 2024-12-03T15:57:43,897 INFO [RS:0;713339a81dd9:42147 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-03T15:57:43,898 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [713339a81dd9,42147,1733241418180] 2024-12-03T15:57:43,899 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/713339a81dd9,42147,1733241418180 already deleted, retry=false 2024-12-03T15:57:43,900 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 713339a81dd9,42147,1733241418180 expired; onlineServers=1 2024-12-03T15:57:43,900 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [713339a81dd9,34819,1733241418314] 2024-12-03T15:57:43,901 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/713339a81dd9,34819,1733241418314 already deleted, retry=false 2024-12-03T15:57:43,901 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 713339a81dd9,34819,1733241418314 expired; onlineServers=0 2024-12-03T15:57:43,901 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '713339a81dd9,34663,1733241417250' ***** 2024-12-03T15:57:43,901 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-12-03T15:57:43,901 INFO [M:0;713339a81dd9:34663 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-03T15:57:43,901 INFO [M:0;713339a81dd9:34663 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-03T15:57:43,901 DEBUG [M:0;713339a81dd9:34663 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-12-03T15:57:43,901 DEBUG [M:0;713339a81dd9:34663 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-12-03T15:57:43,901 DEBUG [master/713339a81dd9:0:becomeActiveMaster-HFileCleaner.large.0-1733241419752 {}] cleaner.HFileCleaner(306): Exit Thread[master/713339a81dd9:0:becomeActiveMaster-HFileCleaner.large.0-1733241419752,5,FailOnTimeoutGroup] 2024-12-03T15:57:43,901 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-12-03T15:57:43,901 DEBUG [master/713339a81dd9:0:becomeActiveMaster-HFileCleaner.small.0-1733241419753 {}] cleaner.HFileCleaner(306): Exit Thread[master/713339a81dd9:0:becomeActiveMaster-HFileCleaner.small.0-1733241419753,5,FailOnTimeoutGroup] 2024-12-03T15:57:43,902 INFO [M:0;713339a81dd9:34663 {}] hbase.ChoreService(370): Chore service for: master/713339a81dd9:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-12-03T15:57:43,902 INFO [M:0;713339a81dd9:34663 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-03T15:57:43,902 DEBUG [M:0;713339a81dd9:34663 {}] master.HMaster(1795): Stopping service threads 2024-12-03T15:57:43,902 INFO [M:0;713339a81dd9:34663 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-12-03T15:57:43,902 INFO [M:0;713339a81dd9:34663 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-03T15:57:43,903 INFO [M:0;713339a81dd9:34663 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-03T15:57:43,903 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-12-03T15:57:43,903 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34663-0x1017926911e0000, quorum=127.0.0.1:61024, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-12-03T15:57:43,903 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34663-0x1017926911e0000, quorum=127.0.0.1:61024, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T15:57:43,904 DEBUG [M:0;713339a81dd9:34663 {}] zookeeper.ZKUtil(347): master:34663-0x1017926911e0000, quorum=127.0.0.1:61024, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-12-03T15:57:43,904 WARN [M:0;713339a81dd9:34663 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-12-03T15:57:43,904 INFO [M:0;713339a81dd9:34663 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:45015/user/jenkins/test-data/c81ce3c5-e5b8-1c2f-869e-120d734cf311/.lastflushedseqids 2024-12-03T15:57:43,918 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46585 is added to blk_1073741941_1121 (size=138) 2024-12-03T15:57:43,919 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40575 is added to blk_1073741941_1121 (size=138) 2024-12-03T15:57:43,919 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44827 is added to blk_1073741941_1121 (size=138) 2024-12-03T15:57:43,920 INFO [M:0;713339a81dd9:34663 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-03T15:57:43,920 INFO [M:0;713339a81dd9:34663 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-12-03T15:57:43,920 DEBUG [M:0;713339a81dd9:34663 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-03T15:57:43,920 INFO [M:0;713339a81dd9:34663 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-03T15:57:43,920 DEBUG [M:0;713339a81dd9:34663 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-03T15:57:43,920 DEBUG [M:0;713339a81dd9:34663 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-03T15:57:43,920 DEBUG [M:0;713339a81dd9:34663 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-03T15:57:43,920 INFO [M:0;713339a81dd9:34663 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=68.33 KB heapSize=83.72 KB 2024-12-03T15:57:43,938 DEBUG [M:0;713339a81dd9:34663 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45015/user/jenkins/test-data/c81ce3c5-e5b8-1c2f-869e-120d734cf311/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/e2f35e9ebfdf4bf5b66e1cb8f22b6f72 is 82, key is hbase:meta,,1/info:regioninfo/1733241420693/Put/seqid=0 2024-12-03T15:57:43,944 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44827 is added to blk_1073741942_1122 (size=5672) 2024-12-03T15:57:43,944 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40575 is added to blk_1073741942_1122 (size=5672) 2024-12-03T15:57:43,946 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46585 is added to blk_1073741942_1122 (size=5672) 2024-12-03T15:57:43,947 INFO [M:0;713339a81dd9:34663 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:45015/user/jenkins/test-data/c81ce3c5-e5b8-1c2f-869e-120d734cf311/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/e2f35e9ebfdf4bf5b66e1cb8f22b6f72 2024-12-03T15:57:43,970 DEBUG [M:0;713339a81dd9:34663 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45015/user/jenkins/test-data/c81ce3c5-e5b8-1c2f-869e-120d734cf311/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/81b3084cbd8d456491749bec16200372 is 1075, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1733241437892/Put/seqid=0 2024-12-03T15:57:43,979 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40575 is added to blk_1073741943_1123 (size=7754) 2024-12-03T15:57:43,980 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44827 is added to blk_1073741943_1123 (size=7754) 2024-12-03T15:57:43,980 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46585 is added to blk_1073741943_1123 (size=7754) 2024-12-03T15:57:43,980 INFO [M:0;713339a81dd9:34663 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=67.60 KB at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:45015/user/jenkins/test-data/c81ce3c5-e5b8-1c2f-869e-120d734cf311/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/81b3084cbd8d456491749bec16200372 2024-12-03T15:57:43,986 INFO [M:0;713339a81dd9:34663 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 81b3084cbd8d456491749bec16200372 2024-12-03T15:57:43,998 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34819-0x1017926911e0002, quorum=127.0.0.1:61024, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-03T15:57:43,998 INFO [RS:1;713339a81dd9:34819 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-03T15:57:43,998 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34819-0x1017926911e0002, quorum=127.0.0.1:61024, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-03T15:57:43,998 INFO [RS:1;713339a81dd9:34819 {}] regionserver.HRegionServer(1031): Exiting; stopping=713339a81dd9,34819,1733241418314; zookeeper connection closed. 2024-12-03T15:57:43,998 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@5bdd66fc {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@5bdd66fc 2024-12-03T15:57:43,999 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42147-0x1017926911e0001, quorum=127.0.0.1:61024, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-03T15:57:43,999 INFO [RS:0;713339a81dd9:42147 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-03T15:57:43,999 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42147-0x1017926911e0001, quorum=127.0.0.1:61024, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-03T15:57:43,999 INFO [RS:0;713339a81dd9:42147 {}] regionserver.HRegionServer(1031): Exiting; stopping=713339a81dd9,42147,1733241418180; zookeeper connection closed. 2024-12-03T15:57:43,999 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@50701c21 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@50701c21 2024-12-03T15:57:44,000 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 3 regionserver(s) complete 2024-12-03T15:57:44,002 DEBUG [M:0;713339a81dd9:34663 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45015/user/jenkins/test-data/c81ce3c5-e5b8-1c2f-869e-120d734cf311/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/406fff0477ca41b6aa8d6245616baaa0 is 69, key is 713339a81dd9,34819,1733241418314/rs:state/1733241419842/Put/seqid=0 2024-12-03T15:57:44,009 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40575 is added to blk_1073741944_1124 (size=5445) 2024-12-03T15:57:44,009 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44827 is added to blk_1073741944_1124 (size=5445) 2024-12-03T15:57:44,010 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46585 is added to blk_1073741944_1124 (size=5445) 2024-12-03T15:57:44,010 INFO [M:0;713339a81dd9:34663 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=249 B at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:45015/user/jenkins/test-data/c81ce3c5-e5b8-1c2f-869e-120d734cf311/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/406fff0477ca41b6aa8d6245616baaa0 2024-12-03T15:57:44,016 INFO [M:0;713339a81dd9:34663 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 406fff0477ca41b6aa8d6245616baaa0 2024-12-03T15:57:44,018 DEBUG [M:0;713339a81dd9:34663 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45015/user/jenkins/test-data/c81ce3c5-e5b8-1c2f-869e-120d734cf311/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/e2f35e9ebfdf4bf5b66e1cb8f22b6f72 as hdfs://localhost:45015/user/jenkins/test-data/c81ce3c5-e5b8-1c2f-869e-120d734cf311/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/e2f35e9ebfdf4bf5b66e1cb8f22b6f72 2024-12-03T15:57:44,023 INFO [M:0;713339a81dd9:34663 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45015/user/jenkins/test-data/c81ce3c5-e5b8-1c2f-869e-120d734cf311/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/e2f35e9ebfdf4bf5b66e1cb8f22b6f72, entries=8, sequenceid=168, filesize=5.5 K 2024-12-03T15:57:44,024 DEBUG [M:0;713339a81dd9:34663 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45015/user/jenkins/test-data/c81ce3c5-e5b8-1c2f-869e-120d734cf311/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/81b3084cbd8d456491749bec16200372 as hdfs://localhost:45015/user/jenkins/test-data/c81ce3c5-e5b8-1c2f-869e-120d734cf311/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/81b3084cbd8d456491749bec16200372 2024-12-03T15:57:44,028 INFO [M:0;713339a81dd9:34663 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 81b3084cbd8d456491749bec16200372 2024-12-03T15:57:44,028 INFO [M:0;713339a81dd9:34663 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45015/user/jenkins/test-data/c81ce3c5-e5b8-1c2f-869e-120d734cf311/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/81b3084cbd8d456491749bec16200372, entries=17, sequenceid=168, filesize=7.6 K 2024-12-03T15:57:44,029 DEBUG [M:0;713339a81dd9:34663 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45015/user/jenkins/test-data/c81ce3c5-e5b8-1c2f-869e-120d734cf311/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/406fff0477ca41b6aa8d6245616baaa0 as hdfs://localhost:45015/user/jenkins/test-data/c81ce3c5-e5b8-1c2f-869e-120d734cf311/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/406fff0477ca41b6aa8d6245616baaa0 2024-12-03T15:57:44,035 INFO [M:0;713339a81dd9:34663 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 406fff0477ca41b6aa8d6245616baaa0 2024-12-03T15:57:44,035 INFO [M:0;713339a81dd9:34663 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45015/user/jenkins/test-data/c81ce3c5-e5b8-1c2f-869e-120d734cf311/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/406fff0477ca41b6aa8d6245616baaa0, entries=3, sequenceid=168, filesize=5.3 K 2024-12-03T15:57:44,036 INFO [M:0;713339a81dd9:34663 {}] regionserver.HRegion(3140): Finished flush of dataSize ~68.33 KB/69972, heapSize ~83.42 KB/85424, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 116ms, sequenceid=168, compaction requested=false 2024-12-03T15:57:44,038 INFO [M:0;713339a81dd9:34663 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-03T15:57:44,038 DEBUG [M:0;713339a81dd9:34663 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733241463920Disabling compacts and flushes for region at 1733241463920Disabling writes for close at 1733241463920Obtaining lock to block concurrent updates at 1733241463920Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1733241463920Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=69972, getHeapSize=85664, getOffHeapSize=0, getCellsCount=195 at 1733241463921 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1733241463921Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1733241463921Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1733241463937 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1733241463938 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1733241463952 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1733241463970 (+18 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1733241463970Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1733241463986 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1733241464002 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1733241464002Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@10edd32: reopening flushed file at 1733241464017 (+15 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3de1687c: reopening flushed file at 1733241464023 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1634ffc4: reopening flushed file at 1733241464029 (+6 ms)Finished flush of dataSize ~68.33 KB/69972, heapSize ~83.42 KB/85424, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 116ms, sequenceid=168, compaction requested=false at 1733241464036 (+7 ms)Writing region close event to WAL at 1733241464038 (+2 ms)Closed at 1733241464038 2024-12-03T15:57:44,041 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40575 is added to blk_1073741830_1006 (size=83594) 2024-12-03T15:57:44,041 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/c81ce3c5-e5b8-1c2f-869e-120d734cf311/MasterData/WALs/713339a81dd9,34663,1733241417250/713339a81dd9%2C34663%2C1733241417250.1733241418917 not finished, retry = 0 2024-12-03T15:57:44,041 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44827 is added to blk_1073741830_1006 (size=83594) 2024-12-03T15:57:44,041 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46585 is added to blk_1073741830_1006 (size=83594) 2024-12-03T15:57:44,142 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-03T15:57:44,142 INFO [M:0;713339a81dd9:34663 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-12-03T15:57:44,142 INFO [M:0;713339a81dd9:34663 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:34663 2024-12-03T15:57:44,143 INFO [M:0;713339a81dd9:34663 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-03T15:57:44,245 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34663-0x1017926911e0000, quorum=127.0.0.1:61024, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-03T15:57:44,245 INFO [M:0;713339a81dd9:34663 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-03T15:57:44,245 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34663-0x1017926911e0000, quorum=127.0.0.1:61024, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-03T15:57:44,252 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733241441470/wal.1733241442014 with renewLeaseKey: DEFAULT_16655 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733241441470/wal.1733241442014 (inode 16655) Holder DFSClient_NONMAPREDUCE_-365845159_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733241441470/wal.1733241442014 (inode 16655) Holder DFSClient_NONMAPREDUCE_-365845159_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy45.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy46.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 25 more 2024-12-03T15:57:44,253 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733241432284/wal.1733241432461 with renewLeaseKey: DEFAULT_16586 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T15:57:44,253 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733241458452/wal.1733241458944 with renewLeaseKey: DEFAULT_16767 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T15:57:44,256 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733241442291/wal.1733241451894 with renewLeaseKey: DEFAULT_16678 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733241442291/wal.1733241451894 (inode 16678) Holder DFSClient_NONMAPREDUCE_-365845159_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733241442291/wal.1733241451894 (inode 16678) Holder DFSClient_NONMAPREDUCE_-365845159_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy45.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy46.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 25 more 2024-12-03T15:57:44,257 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733241423446/wal.1733241423546 with renewLeaseKey: DEFAULT_16506 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T15:57:44,259 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testdatalosswheninputerror-manual,16010,1733241452253/wal.1733241452971 with renewLeaseKey: DEFAULT_16704 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testdatalosswheninputerror-manual,16010,1733241452253/wal.1733241452971 (inode 16704) Holder DFSClient_NONMAPREDUCE_-365845159_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testdatalosswheninputerror-manual,16010,1733241452253/wal.1733241452971 (inode 16704) Holder DFSClient_NONMAPREDUCE_-365845159_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy45.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy46.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 25 more 2024-12-03T15:57:44,259 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733241453269/wal.1733241453433 with renewLeaseKey: DEFAULT_16726 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T15:57:44,262 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testnameconflictwhensplit1-manual,16010,1733241422283/wal.1733241423256 with renewLeaseKey: DEFAULT_16485 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testnameconflictwhensplit1-manual,16010,1733241422283/wal.1733241423256 (inode 16485) Holder DFSClient_NONMAPREDUCE_-365845159_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testnameconflictwhensplit1-manual,16010,1733241422283/wal.1733241423256 (inode 16485) Holder DFSClient_NONMAPREDUCE_-365845159_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy45.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy46.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 25 more 2024-12-03T15:57:44,264 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testnameconflictwhensplit0-manual,16010,1733241421166/wal.1733241421888 with renewLeaseKey: DEFAULT_16462 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testnameconflictwhensplit0-manual,16010,1733241421166/wal.1733241421888 (inode 16462) Holder DFSClient_NONMAPREDUCE_-365845159_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testnameconflictwhensplit0-manual,16010,1733241421166/wal.1733241421888 (inode 16462) Holder DFSClient_NONMAPREDUCE_-365845159_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy45.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy46.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 25 more 2024-12-03T15:57:44,268 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@420bc738{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-03T15:57:44,271 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@5546381a{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-03T15:57:44,271 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-03T15:57:44,271 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@228ffa29{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-03T15:57:44,271 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1e1a8b20{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb3fe2df-6726-509d-596d-17caab0e5978/hadoop.log.dir/,STOPPED} 2024-12-03T15:57:44,274 WARN [BP-1122350750-172.17.0.3-1733241413817 heartbeating to localhost/127.0.0.1:45015 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-03T15:57:44,274 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-03T15:57:44,274 WARN [BP-1122350750-172.17.0.3-1733241413817 heartbeating to localhost/127.0.0.1:45015 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1122350750-172.17.0.3-1733241413817 (Datanode Uuid 5360fc40-fc97-40f7-a77d-73c889ffe94e) service to localhost/127.0.0.1:45015 2024-12-03T15:57:44,274 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-03T15:57:44,276 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb3fe2df-6726-509d-596d-17caab0e5978/cluster_ef20cb83-4e1e-0871-5945-558338f0ff60/data/data5/current/BP-1122350750-172.17.0.3-1733241413817 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-03T15:57:44,276 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb3fe2df-6726-509d-596d-17caab0e5978/cluster_ef20cb83-4e1e-0871-5945-558338f0ff60/data/data6/current/BP-1122350750-172.17.0.3-1733241413817 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-03T15:57:44,277 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-03T15:57:44,279 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@64052af6{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-03T15:57:44,279 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@645d24a7{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-03T15:57:44,279 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-03T15:57:44,280 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7c7d32f8{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-03T15:57:44,280 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7acb2c1e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb3fe2df-6726-509d-596d-17caab0e5978/hadoop.log.dir/,STOPPED} 2024-12-03T15:57:44,281 WARN [BP-1122350750-172.17.0.3-1733241413817 heartbeating to localhost/127.0.0.1:45015 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-03T15:57:44,281 WARN [BP-1122350750-172.17.0.3-1733241413817 heartbeating to localhost/127.0.0.1:45015 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1122350750-172.17.0.3-1733241413817 (Datanode Uuid 98c3b8bb-8b94-459f-a46e-c24ee437a325) service to localhost/127.0.0.1:45015 2024-12-03T15:57:44,281 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-03T15:57:44,281 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-03T15:57:44,282 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb3fe2df-6726-509d-596d-17caab0e5978/cluster_ef20cb83-4e1e-0871-5945-558338f0ff60/data/data3/current/BP-1122350750-172.17.0.3-1733241413817 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-03T15:57:44,282 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb3fe2df-6726-509d-596d-17caab0e5978/cluster_ef20cb83-4e1e-0871-5945-558338f0ff60/data/data4/current/BP-1122350750-172.17.0.3-1733241413817 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-03T15:57:44,282 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-03T15:57:44,284 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@501095cc{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-03T15:57:44,285 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@2b37aa87{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-03T15:57:44,285 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-03T15:57:44,285 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@751d2fa4{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-03T15:57:44,285 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3161f7a1{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb3fe2df-6726-509d-596d-17caab0e5978/hadoop.log.dir/,STOPPED} 2024-12-03T15:57:44,286 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-03T15:57:44,286 WARN [BP-1122350750-172.17.0.3-1733241413817 heartbeating to localhost/127.0.0.1:45015 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-03T15:57:44,286 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-03T15:57:44,286 WARN [BP-1122350750-172.17.0.3-1733241413817 heartbeating to localhost/127.0.0.1:45015 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1122350750-172.17.0.3-1733241413817 (Datanode Uuid b7a8f304-74d8-4a03-ac93-6ac0859d29b6) service to localhost/127.0.0.1:45015 2024-12-03T15:57:44,287 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb3fe2df-6726-509d-596d-17caab0e5978/cluster_ef20cb83-4e1e-0871-5945-558338f0ff60/data/data2/current/BP-1122350750-172.17.0.3-1733241413817 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-03T15:57:44,287 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb3fe2df-6726-509d-596d-17caab0e5978/cluster_ef20cb83-4e1e-0871-5945-558338f0ff60/data/data1/current/BP-1122350750-172.17.0.3-1733241413817 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-03T15:57:44,287 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-03T15:57:44,295 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4c03e044{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-03T15:57:44,296 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@698721f1{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-03T15:57:44,296 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-03T15:57:44,296 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@652099c6{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-03T15:57:44,296 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@264e890{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb3fe2df-6726-509d-596d-17caab0e5978/hadoop.log.dir/,STOPPED} 2024-12-03T15:57:44,307 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-12-03T15:57:44,359 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down