2024-11-14 14:51:48,261 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@20b2475a 2024-11-14 14:51:48,271 main DEBUG Took 0.008930 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-14 14:51:48,272 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-14 14:51:48,272 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-14 14:51:48,273 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-14 14:51:48,274 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 14:51:48,280 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-14 14:51:48,292 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 14:51:48,293 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 14:51:48,294 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 14:51:48,294 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 14:51:48,295 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 14:51:48,295 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 14:51:48,296 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 14:51:48,296 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 14:51:48,296 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 14:51:48,297 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 14:51:48,297 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 14:51:48,298 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 14:51:48,298 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 14:51:48,298 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 14:51:48,299 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 14:51:48,299 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 14:51:48,300 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 14:51:48,300 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 14:51:48,300 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 14:51:48,301 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 14:51:48,301 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 14:51:48,301 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 14:51:48,302 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 14:51:48,302 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 14:51:48,303 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 14:51:48,303 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-14 14:51:48,304 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 14:51:48,305 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-14 14:51:48,307 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-14 14:51:48,307 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-14 14:51:48,308 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-14 14:51:48,309 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-14 14:51:48,317 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-14 14:51:48,319 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-14 14:51:48,321 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-14 14:51:48,321 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-14 14:51:48,321 main DEBUG createAppenders(={Console}) 2024-11-14 14:51:48,322 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@20b2475a initialized 2024-11-14 14:51:48,322 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@20b2475a 2024-11-14 14:51:48,322 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@20b2475a OK. 2024-11-14 14:51:48,323 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-14 14:51:48,323 main DEBUG OutputStream closed 2024-11-14 14:51:48,323 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-14 14:51:48,324 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-14 14:51:48,324 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@4310d43 OK 2024-11-14 14:51:48,403 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-14 14:51:48,407 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-14 14:51:48,408 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-14 14:51:48,410 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-14 14:51:48,410 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-14 14:51:48,411 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-14 14:51:48,411 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-14 14:51:48,412 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-14 14:51:48,412 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-14 14:51:48,413 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-14 14:51:48,414 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-14 14:51:48,414 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-14 14:51:48,415 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-14 14:51:48,415 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-14 14:51:48,415 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-14 14:51:48,416 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-14 14:51:48,416 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-14 14:51:48,418 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-14 14:51:48,420 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-14 14:51:48,421 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@12f9af83) with optional ClassLoader: null 2024-11-14 14:51:48,421 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-14 14:51:48,422 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@12f9af83] started OK. 2024-11-14T14:51:48,752 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c843459-ed19-f974-f761-d73dee21a91b 2024-11-14 14:51:48,755 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-14 14:51:48,755 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-14T14:51:48,764 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay timeout: 13 mins 2024-11-14T14:51:48,770 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.regionserver.wal.TestSecureAsyncWALReplay timeout: 13 mins 2024-11-14T14:51:48,791 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-14T14:51:48,838 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-14T14:51:48,839 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-14T14:51:48,853 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=3, rsPorts=, rsClass=null, numDataNodes=3, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-14T14:51:48,874 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c843459-ed19-f974-f761-d73dee21a91b/cluster_fceed6e6-d3b5-652e-82f8-dddc2c9b7e2b, deleteOnExit=true 2024-11-14T14:51:48,874 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-14T14:51:48,876 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c843459-ed19-f974-f761-d73dee21a91b/test.cache.data in system properties and HBase conf 2024-11-14T14:51:48,876 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c843459-ed19-f974-f761-d73dee21a91b/hadoop.tmp.dir in system properties and HBase conf 2024-11-14T14:51:48,877 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c843459-ed19-f974-f761-d73dee21a91b/hadoop.log.dir in system properties and HBase conf 2024-11-14T14:51:48,878 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c843459-ed19-f974-f761-d73dee21a91b/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-14T14:51:48,878 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c843459-ed19-f974-f761-d73dee21a91b/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-14T14:51:48,879 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-14T14:51:48,976 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-14T14:51:49,077 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-14T14:51:49,082 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c843459-ed19-f974-f761-d73dee21a91b/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-14T14:51:49,083 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c843459-ed19-f974-f761-d73dee21a91b/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-14T14:51:49,084 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c843459-ed19-f974-f761-d73dee21a91b/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-14T14:51:49,085 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c843459-ed19-f974-f761-d73dee21a91b/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-14T14:51:49,085 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c843459-ed19-f974-f761-d73dee21a91b/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-14T14:51:49,086 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c843459-ed19-f974-f761-d73dee21a91b/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-14T14:51:49,086 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c843459-ed19-f974-f761-d73dee21a91b/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-14T14:51:49,087 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c843459-ed19-f974-f761-d73dee21a91b/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-14T14:51:49,088 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c843459-ed19-f974-f761-d73dee21a91b/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-14T14:51:49,088 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c843459-ed19-f974-f761-d73dee21a91b/nfs.dump.dir in system properties and HBase conf 2024-11-14T14:51:49,089 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c843459-ed19-f974-f761-d73dee21a91b/java.io.tmpdir in system properties and HBase conf 2024-11-14T14:51:49,089 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c843459-ed19-f974-f761-d73dee21a91b/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-14T14:51:49,090 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c843459-ed19-f974-f761-d73dee21a91b/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-14T14:51:49,091 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c843459-ed19-f974-f761-d73dee21a91b/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-14T14:51:49,954 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-14T14:51:50,048 INFO [Time-limited test {}] log.Log(170): Logging initialized @2583ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-14T14:51:50,137 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-14T14:51:50,206 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-14T14:51:50,235 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-14T14:51:50,235 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-14T14:51:50,237 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-14T14:51:50,249 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-14T14:51:50,252 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@264e890{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c843459-ed19-f974-f761-d73dee21a91b/hadoop.log.dir/,AVAILABLE} 2024-11-14T14:51:50,253 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@652099c6{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-14T14:51:50,433 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4c03e044{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c843459-ed19-f974-f761-d73dee21a91b/java.io.tmpdir/jetty-localhost-35641-hadoop-hdfs-3_4_1-tests_jar-_-any-12113260208007529127/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-14T14:51:50,440 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@698721f1{HTTP/1.1, (http/1.1)}{localhost:35641} 2024-11-14T14:51:50,440 INFO [Time-limited test {}] server.Server(415): Started @2976ms 2024-11-14T14:51:50,853 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-14T14:51:50,862 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-14T14:51:50,864 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-14T14:51:50,864 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-14T14:51:50,864 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-14T14:51:50,865 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@cf5a85e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c843459-ed19-f974-f761-d73dee21a91b/hadoop.log.dir/,AVAILABLE} 2024-11-14T14:51:50,865 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4a359997{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-14T14:51:50,989 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@330740de{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c843459-ed19-f974-f761-d73dee21a91b/java.io.tmpdir/jetty-localhost-33153-hadoop-hdfs-3_4_1-tests_jar-_-any-16869721493274200616/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-14T14:51:50,989 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@7b24cab9{HTTP/1.1, (http/1.1)}{localhost:33153} 2024-11-14T14:51:50,990 INFO [Time-limited test {}] server.Server(415): Started @3525ms 2024-11-14T14:51:51,053 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-14T14:51:51,179 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-14T14:51:51,187 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-14T14:51:51,189 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-14T14:51:51,189 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-14T14:51:51,189 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-14T14:51:51,194 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@46b092e1{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c843459-ed19-f974-f761-d73dee21a91b/hadoop.log.dir/,AVAILABLE} 2024-11-14T14:51:51,195 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@5cc2d6b{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-14T14:51:51,330 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@7bd427b8{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c843459-ed19-f974-f761-d73dee21a91b/java.io.tmpdir/jetty-localhost-34239-hadoop-hdfs-3_4_1-tests_jar-_-any-74730189831434424/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-14T14:51:51,331 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@6915083f{HTTP/1.1, (http/1.1)}{localhost:34239} 2024-11-14T14:51:51,331 INFO [Time-limited test {}] server.Server(415): Started @3866ms 2024-11-14T14:51:51,333 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-14T14:51:51,375 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-14T14:51:51,380 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-14T14:51:51,385 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-14T14:51:51,385 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-14T14:51:51,385 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-14T14:51:51,386 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@2a6d5e13{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c843459-ed19-f974-f761-d73dee21a91b/hadoop.log.dir/,AVAILABLE} 2024-11-14T14:51:51,386 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@f9972d0{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-14T14:51:51,487 WARN [Thread-108 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c843459-ed19-f974-f761-d73dee21a91b/cluster_fceed6e6-d3b5-652e-82f8-dddc2c9b7e2b/data/data4/current/BP-1055376130-172.17.0.2-1731595909704/current, will proceed with Du for space computation calculation, 2024-11-14T14:51:51,487 WARN [Thread-105 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c843459-ed19-f974-f761-d73dee21a91b/cluster_fceed6e6-d3b5-652e-82f8-dddc2c9b7e2b/data/data1/current/BP-1055376130-172.17.0.2-1731595909704/current, will proceed with Du for space computation calculation, 2024-11-14T14:51:51,487 WARN [Thread-107 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c843459-ed19-f974-f761-d73dee21a91b/cluster_fceed6e6-d3b5-652e-82f8-dddc2c9b7e2b/data/data3/current/BP-1055376130-172.17.0.2-1731595909704/current, will proceed with Du for space computation calculation, 2024-11-14T14:51:51,487 WARN [Thread-106 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c843459-ed19-f974-f761-d73dee21a91b/cluster_fceed6e6-d3b5-652e-82f8-dddc2c9b7e2b/data/data2/current/BP-1055376130-172.17.0.2-1731595909704/current, will proceed with Du for space computation calculation, 2024-11-14T14:51:51,519 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@35f1150e{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c843459-ed19-f974-f761-d73dee21a91b/java.io.tmpdir/jetty-localhost-45255-hadoop-hdfs-3_4_1-tests_jar-_-any-6460849406129448781/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-14T14:51:51,520 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@13a77e13{HTTP/1.1, (http/1.1)}{localhost:45255} 2024-11-14T14:51:51,521 INFO [Time-limited test {}] server.Server(415): Started @4056ms 2024-11-14T14:51:51,523 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-14T14:51:51,538 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-14T14:51:51,538 WARN [Thread-82 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-14T14:51:51,612 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x42f4e1c15fa3c9c7 with lease ID 0x36fdbfcfb91132f4: Processing first storage report for DS-913adb4a-7946-4f46-896c-56bcdc4fcfa8 from datanode DatanodeRegistration(127.0.0.1:44549, datanodeUuid=63d3b466-1d56-4787-911e-c33e6d4192c1, infoPort=33515, infoSecurePort=0, ipcPort=34459, storageInfo=lv=-57;cid=testClusterID;nsid=1885725529;c=1731595909704) 2024-11-14T14:51:51,614 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x42f4e1c15fa3c9c7 with lease ID 0x36fdbfcfb91132f4: from storage DS-913adb4a-7946-4f46-896c-56bcdc4fcfa8 node DatanodeRegistration(127.0.0.1:44549, datanodeUuid=63d3b466-1d56-4787-911e-c33e6d4192c1, infoPort=33515, infoSecurePort=0, ipcPort=34459, storageInfo=lv=-57;cid=testClusterID;nsid=1885725529;c=1731595909704), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-14T14:51:51,614 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xd592c1ab09a8518 with lease ID 0x36fdbfcfb91132f5: Processing first storage report for DS-5c245456-9209-4937-bfad-43a9a1b09820 from datanode DatanodeRegistration(127.0.0.1:36189, datanodeUuid=d55da410-599d-40d9-9c9f-2653bb81e8cf, infoPort=33623, infoSecurePort=0, ipcPort=39351, storageInfo=lv=-57;cid=testClusterID;nsid=1885725529;c=1731595909704) 2024-11-14T14:51:51,615 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xd592c1ab09a8518 with lease ID 0x36fdbfcfb91132f5: from storage DS-5c245456-9209-4937-bfad-43a9a1b09820 node DatanodeRegistration(127.0.0.1:36189, datanodeUuid=d55da410-599d-40d9-9c9f-2653bb81e8cf, infoPort=33623, infoSecurePort=0, ipcPort=39351, storageInfo=lv=-57;cid=testClusterID;nsid=1885725529;c=1731595909704), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-14T14:51:51,615 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x42f4e1c15fa3c9c7 with lease ID 0x36fdbfcfb91132f4: Processing first storage report for DS-ef570fc3-951e-4cb6-87b7-d02c411d497c from datanode DatanodeRegistration(127.0.0.1:44549, datanodeUuid=63d3b466-1d56-4787-911e-c33e6d4192c1, infoPort=33515, infoSecurePort=0, ipcPort=34459, storageInfo=lv=-57;cid=testClusterID;nsid=1885725529;c=1731595909704) 2024-11-14T14:51:51,615 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x42f4e1c15fa3c9c7 with lease ID 0x36fdbfcfb91132f4: from storage DS-ef570fc3-951e-4cb6-87b7-d02c411d497c node DatanodeRegistration(127.0.0.1:44549, datanodeUuid=63d3b466-1d56-4787-911e-c33e6d4192c1, infoPort=33515, infoSecurePort=0, ipcPort=34459, storageInfo=lv=-57;cid=testClusterID;nsid=1885725529;c=1731595909704), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-14T14:51:51,616 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xd592c1ab09a8518 with lease ID 0x36fdbfcfb91132f5: Processing first storage report for DS-8a22c6d4-064d-41f8-80d2-fc3fd88dd3e2 from datanode DatanodeRegistration(127.0.0.1:36189, datanodeUuid=d55da410-599d-40d9-9c9f-2653bb81e8cf, infoPort=33623, infoSecurePort=0, ipcPort=39351, storageInfo=lv=-57;cid=testClusterID;nsid=1885725529;c=1731595909704) 2024-11-14T14:51:51,616 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xd592c1ab09a8518 with lease ID 0x36fdbfcfb91132f5: from storage DS-8a22c6d4-064d-41f8-80d2-fc3fd88dd3e2 node DatanodeRegistration(127.0.0.1:36189, datanodeUuid=d55da410-599d-40d9-9c9f-2653bb81e8cf, infoPort=33623, infoSecurePort=0, ipcPort=39351, storageInfo=lv=-57;cid=testClusterID;nsid=1885725529;c=1731595909704), blocks: 0, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-14T14:51:51,640 WARN [Thread-140 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c843459-ed19-f974-f761-d73dee21a91b/cluster_fceed6e6-d3b5-652e-82f8-dddc2c9b7e2b/data/data6/current/BP-1055376130-172.17.0.2-1731595909704/current, will proceed with Du for space computation calculation, 2024-11-14T14:51:51,640 WARN [Thread-139 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c843459-ed19-f974-f761-d73dee21a91b/cluster_fceed6e6-d3b5-652e-82f8-dddc2c9b7e2b/data/data5/current/BP-1055376130-172.17.0.2-1731595909704/current, will proceed with Du for space computation calculation, 2024-11-14T14:51:51,684 WARN [Thread-129 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-14T14:51:51,689 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xb101cf1e37767ea7 with lease ID 0x36fdbfcfb91132f6: Processing first storage report for DS-c1b8ccdc-5e2c-4b3d-bb1e-6941b40dfb7e from datanode DatanodeRegistration(127.0.0.1:42819, datanodeUuid=c57b7594-0e99-485a-938c-81b59bea0cf1, infoPort=34777, infoSecurePort=0, ipcPort=38967, storageInfo=lv=-57;cid=testClusterID;nsid=1885725529;c=1731595909704) 2024-11-14T14:51:51,689 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xb101cf1e37767ea7 with lease ID 0x36fdbfcfb91132f6: from storage DS-c1b8ccdc-5e2c-4b3d-bb1e-6941b40dfb7e node DatanodeRegistration(127.0.0.1:42819, datanodeUuid=c57b7594-0e99-485a-938c-81b59bea0cf1, infoPort=34777, infoSecurePort=0, ipcPort=38967, storageInfo=lv=-57;cid=testClusterID;nsid=1885725529;c=1731595909704), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-14T14:51:51,689 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xb101cf1e37767ea7 with lease ID 0x36fdbfcfb91132f6: Processing first storage report for DS-3c0e6d17-49c2-42cd-8f57-a0969152b409 from datanode DatanodeRegistration(127.0.0.1:42819, datanodeUuid=c57b7594-0e99-485a-938c-81b59bea0cf1, infoPort=34777, infoSecurePort=0, ipcPort=38967, storageInfo=lv=-57;cid=testClusterID;nsid=1885725529;c=1731595909704) 2024-11-14T14:51:51,690 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xb101cf1e37767ea7 with lease ID 0x36fdbfcfb91132f6: from storage DS-3c0e6d17-49c2-42cd-8f57-a0969152b409 node DatanodeRegistration(127.0.0.1:42819, datanodeUuid=c57b7594-0e99-485a-938c-81b59bea0cf1, infoPort=34777, infoSecurePort=0, ipcPort=38967, storageInfo=lv=-57;cid=testClusterID;nsid=1885725529;c=1731595909704), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-14T14:51:51,887 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c843459-ed19-f974-f761-d73dee21a91b 2024-11-14T14:51:51,984 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c843459-ed19-f974-f761-d73dee21a91b/cluster_fceed6e6-d3b5-652e-82f8-dddc2c9b7e2b/zookeeper_0, clientPort=51231, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c843459-ed19-f974-f761-d73dee21a91b/cluster_fceed6e6-d3b5-652e-82f8-dddc2c9b7e2b/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c843459-ed19-f974-f761-d73dee21a91b/cluster_fceed6e6-d3b5-652e-82f8-dddc2c9b7e2b/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-14T14:51:51,996 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=51231 2024-11-14T14:51:52,006 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-14T14:51:52,008 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-14T14:51:52,217 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36189 is added to blk_1073741825_1001 (size=7) 2024-11-14T14:51:52,218 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42819 is added to blk_1073741825_1001 (size=7) 2024-11-14T14:51:52,218 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44549 is added to blk_1073741825_1001 (size=7) 2024-11-14T14:51:52,625 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:34445/user/jenkins/test-data/35161b07-018a-0344-8c43-1519f3812567 with version=8 2024-11-14T14:51:52,625 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:34445/user/jenkins/test-data/35161b07-018a-0344-8c43-1519f3812567/hbase-staging 2024-11-14T14:51:52,935 INFO [Time-limited test {}] client.ConnectionUtils(128): master/da1c6afcd1f9:0 server-side Connection retries=45 2024-11-14T14:51:52,947 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-14T14:51:52,947 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-14T14:51:52,952 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-14T14:51:52,952 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-14T14:51:52,952 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-14T14:51:53,092 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-14T14:51:53,156 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-14T14:51:53,164 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-14T14:51:53,168 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-14T14:51:53,197 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 4000 (auto-detected) 2024-11-14T14:51:53,198 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-14T14:51:53,218 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:42151 2024-11-14T14:51:53,238 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:42151 connecting to ZooKeeper ensemble=127.0.0.1:51231 2024-11-14T14:51:53,269 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:421510x0, quorum=127.0.0.1:51231, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-14T14:51:53,272 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:42151-0x101a74438830000 connected 2024-11-14T14:51:53,297 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-14T14:51:53,301 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-14T14:51:53,310 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:42151-0x101a74438830000, quorum=127.0.0.1:51231, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-14T14:51:53,315 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:34445/user/jenkins/test-data/35161b07-018a-0344-8c43-1519f3812567, hbase.cluster.distributed=false 2024-11-14T14:51:53,344 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:42151-0x101a74438830000, quorum=127.0.0.1:51231, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-14T14:51:53,350 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=42151 2024-11-14T14:51:53,350 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=42151 2024-11-14T14:51:53,351 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=42151 2024-11-14T14:51:53,351 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=42151 2024-11-14T14:51:53,352 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=42151 2024-11-14T14:51:53,469 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/da1c6afcd1f9:0 server-side Connection retries=45 2024-11-14T14:51:53,471 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-14T14:51:53,472 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-14T14:51:53,472 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-14T14:51:53,472 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-14T14:51:53,472 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-14T14:51:53,475 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-14T14:51:53,477 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-14T14:51:53,478 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:37695 2024-11-14T14:51:53,480 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:37695 connecting to ZooKeeper ensemble=127.0.0.1:51231 2024-11-14T14:51:53,481 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-14T14:51:53,485 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-14T14:51:53,492 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:376950x0, quorum=127.0.0.1:51231, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-14T14:51:53,493 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:37695-0x101a74438830001 connected 2024-11-14T14:51:53,493 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:37695-0x101a74438830001, quorum=127.0.0.1:51231, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-14T14:51:53,497 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-14T14:51:53,505 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-14T14:51:53,508 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:37695-0x101a74438830001, quorum=127.0.0.1:51231, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-14T14:51:53,512 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:37695-0x101a74438830001, quorum=127.0.0.1:51231, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-14T14:51:53,513 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=37695 2024-11-14T14:51:53,514 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=37695 2024-11-14T14:51:53,514 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=37695 2024-11-14T14:51:53,515 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=37695 2024-11-14T14:51:53,515 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=37695 2024-11-14T14:51:53,533 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/da1c6afcd1f9:0 server-side Connection retries=45 2024-11-14T14:51:53,533 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-14T14:51:53,534 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-14T14:51:53,534 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-14T14:51:53,534 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-14T14:51:53,534 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-14T14:51:53,535 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-14T14:51:53,535 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-14T14:51:53,536 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:34155 2024-11-14T14:51:53,538 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:34155 connecting to ZooKeeper ensemble=127.0.0.1:51231 2024-11-14T14:51:53,540 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-14T14:51:53,543 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-14T14:51:53,550 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:341550x0, quorum=127.0.0.1:51231, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-14T14:51:53,550 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:34155-0x101a74438830002 connected 2024-11-14T14:51:53,551 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:34155-0x101a74438830002, quorum=127.0.0.1:51231, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-14T14:51:53,551 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-14T14:51:53,552 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-14T14:51:53,554 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:34155-0x101a74438830002, quorum=127.0.0.1:51231, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-14T14:51:53,556 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:34155-0x101a74438830002, quorum=127.0.0.1:51231, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-14T14:51:53,558 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=34155 2024-11-14T14:51:53,560 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=34155 2024-11-14T14:51:53,561 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=34155 2024-11-14T14:51:53,563 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=34155 2024-11-14T14:51:53,564 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=34155 2024-11-14T14:51:53,586 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/da1c6afcd1f9:0 server-side Connection retries=45 2024-11-14T14:51:53,587 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-14T14:51:53,587 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-14T14:51:53,587 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-14T14:51:53,587 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-14T14:51:53,588 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-14T14:51:53,588 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-14T14:51:53,588 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-14T14:51:53,589 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:46007 2024-11-14T14:51:53,592 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:46007 connecting to ZooKeeper ensemble=127.0.0.1:51231 2024-11-14T14:51:53,594 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-14T14:51:53,597 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-14T14:51:53,603 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:460070x0, quorum=127.0.0.1:51231, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-14T14:51:53,604 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:46007-0x101a74438830003 connected 2024-11-14T14:51:53,604 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:46007-0x101a74438830003, quorum=127.0.0.1:51231, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-14T14:51:53,605 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-14T14:51:53,606 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-14T14:51:53,607 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:46007-0x101a74438830003, quorum=127.0.0.1:51231, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-14T14:51:53,609 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:46007-0x101a74438830003, quorum=127.0.0.1:51231, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-14T14:51:53,610 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=46007 2024-11-14T14:51:53,611 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=46007 2024-11-14T14:51:53,612 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=46007 2024-11-14T14:51:53,615 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=46007 2024-11-14T14:51:53,615 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=46007 2024-11-14T14:51:53,631 DEBUG [M:0;da1c6afcd1f9:42151 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;da1c6afcd1f9:42151 2024-11-14T14:51:53,632 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/da1c6afcd1f9,42151,1731595912734 2024-11-14T14:51:53,640 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37695-0x101a74438830001, quorum=127.0.0.1:51231, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-14T14:51:53,640 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34155-0x101a74438830002, quorum=127.0.0.1:51231, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-14T14:51:53,640 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46007-0x101a74438830003, quorum=127.0.0.1:51231, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-14T14:51:53,640 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42151-0x101a74438830000, quorum=127.0.0.1:51231, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-14T14:51:53,642 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:42151-0x101a74438830000, quorum=127.0.0.1:51231, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/da1c6afcd1f9,42151,1731595912734 2024-11-14T14:51:53,663 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46007-0x101a74438830003, quorum=127.0.0.1:51231, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-14T14:51:53,663 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42151-0x101a74438830000, quorum=127.0.0.1:51231, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T14:51:53,663 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34155-0x101a74438830002, quorum=127.0.0.1:51231, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-14T14:51:53,663 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37695-0x101a74438830001, quorum=127.0.0.1:51231, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-14T14:51:53,663 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46007-0x101a74438830003, quorum=127.0.0.1:51231, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T14:51:53,663 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34155-0x101a74438830002, quorum=127.0.0.1:51231, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T14:51:53,664 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37695-0x101a74438830001, quorum=127.0.0.1:51231, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T14:51:53,665 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:42151-0x101a74438830000, quorum=127.0.0.1:51231, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-14T14:51:53,666 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/da1c6afcd1f9,42151,1731595912734 from backup master directory 2024-11-14T14:51:53,669 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46007-0x101a74438830003, quorum=127.0.0.1:51231, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-14T14:51:53,669 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37695-0x101a74438830001, quorum=127.0.0.1:51231, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-14T14:51:53,669 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34155-0x101a74438830002, quorum=127.0.0.1:51231, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-14T14:51:53,669 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42151-0x101a74438830000, quorum=127.0.0.1:51231, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/da1c6afcd1f9,42151,1731595912734 2024-11-14T14:51:53,669 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42151-0x101a74438830000, quorum=127.0.0.1:51231, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-14T14:51:53,670 WARN [master/da1c6afcd1f9:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-14T14:51:53,670 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=da1c6afcd1f9,42151,1731595912734 2024-11-14T14:51:53,673 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-14T14:51:53,674 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-14T14:51:53,734 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:34445/user/jenkins/test-data/35161b07-018a-0344-8c43-1519f3812567/hbase.id] with ID: 72d32361-5e69-4ee6-9ad5-db998b6b568a 2024-11-14T14:51:53,735 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:34445/user/jenkins/test-data/35161b07-018a-0344-8c43-1519f3812567/.tmp/hbase.id 2024-11-14T14:51:53,748 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44549 is added to blk_1073741826_1002 (size=42) 2024-11-14T14:51:53,749 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42819 is added to blk_1073741826_1002 (size=42) 2024-11-14T14:51:53,749 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36189 is added to blk_1073741826_1002 (size=42) 2024-11-14T14:51:53,750 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:34445/user/jenkins/test-data/35161b07-018a-0344-8c43-1519f3812567/.tmp/hbase.id]:[hdfs://localhost:34445/user/jenkins/test-data/35161b07-018a-0344-8c43-1519f3812567/hbase.id] 2024-11-14T14:51:53,795 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-14T14:51:53,800 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-14T14:51:53,820 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 18ms. 2024-11-14T14:51:53,824 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42151-0x101a74438830000, quorum=127.0.0.1:51231, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T14:51:53,824 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46007-0x101a74438830003, quorum=127.0.0.1:51231, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T14:51:53,824 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37695-0x101a74438830001, quorum=127.0.0.1:51231, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T14:51:53,824 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34155-0x101a74438830002, quorum=127.0.0.1:51231, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T14:51:53,842 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44549 is added to blk_1073741827_1003 (size=196) 2024-11-14T14:51:53,843 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36189 is added to blk_1073741827_1003 (size=196) 2024-11-14T14:51:53,844 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42819 is added to blk_1073741827_1003 (size=196) 2024-11-14T14:51:53,861 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-14T14:51:53,863 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-14T14:51:53,870 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-14T14:51:53,906 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42819 is added to blk_1073741828_1004 (size=1189) 2024-11-14T14:51:53,906 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36189 is added to blk_1073741828_1004 (size=1189) 2024-11-14T14:51:53,907 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44549 is added to blk_1073741828_1004 (size=1189) 2024-11-14T14:51:53,927 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:34445/user/jenkins/test-data/35161b07-018a-0344-8c43-1519f3812567/MasterData/data/master/store 2024-11-14T14:51:53,950 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44549 is added to blk_1073741829_1005 (size=34) 2024-11-14T14:51:53,951 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36189 is added to blk_1073741829_1005 (size=34) 2024-11-14T14:51:53,951 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42819 is added to blk_1073741829_1005 (size=34) 2024-11-14T14:51:53,958 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-14T14:51:53,962 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T14:51:53,964 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-14T14:51:53,964 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-14T14:51:53,964 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-14T14:51:53,966 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-14T14:51:53,967 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-14T14:51:53,967 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-14T14:51:53,969 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731595913964Disabling compacts and flushes for region at 1731595913964Disabling writes for close at 1731595913967 (+3 ms)Writing region close event to WAL at 1731595913967Closed at 1731595913967 2024-11-14T14:51:53,971 WARN [master/da1c6afcd1f9:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:34445/user/jenkins/test-data/35161b07-018a-0344-8c43-1519f3812567/MasterData/data/master/store/.initializing 2024-11-14T14:51:53,971 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:34445/user/jenkins/test-data/35161b07-018a-0344-8c43-1519f3812567/MasterData/WALs/da1c6afcd1f9,42151,1731595912734 2024-11-14T14:51:53,981 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-14T14:51:53,999 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=da1c6afcd1f9%2C42151%2C1731595912734, suffix=, logDir=hdfs://localhost:34445/user/jenkins/test-data/35161b07-018a-0344-8c43-1519f3812567/MasterData/WALs/da1c6afcd1f9,42151,1731595912734, archiveDir=hdfs://localhost:34445/user/jenkins/test-data/35161b07-018a-0344-8c43-1519f3812567/MasterData/oldWALs, maxLogs=10 2024-11-14T14:51:54,033 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/35161b07-018a-0344-8c43-1519f3812567/MasterData/WALs/da1c6afcd1f9,42151,1731595912734/da1c6afcd1f9%2C42151%2C1731595912734.1731595914006, exclude list is [], retry=0 2024-11-14T14:51:54,054 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.initialize(FanOutOneBlockAsyncDFSOutputHelper.java:413) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$5.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:472) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$5.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:467) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.NettyFutureUtils.lambda$addListener$0(NettyFutureUtils.java:56) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setSuccess0(DefaultPromise.java:625) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.trySuccess(DefaultPromise.java:105) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPromise.trySuccess(DefaultChannelPromise.java:84) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:658) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:696) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:51:54,059 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36189,DS-5c245456-9209-4937-bfad-43a9a1b09820,DISK] 2024-11-14T14:51:54,059 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44549,DS-913adb4a-7946-4f46-896c-56bcdc4fcfa8,DISK] 2024-11-14T14:51:54,059 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42819,DS-c1b8ccdc-5e2c-4b3d-bb1e-6941b40dfb7e,DISK] 2024-11-14T14:51:54,063 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-14T14:51:54,097 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] crypto.Encryption(565): Installed org.apache.hadoop.hbase.io.crypto.MockAesKeyProvider into key provider cache 2024-11-14T14:51:54,165 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/35161b07-018a-0344-8c43-1519f3812567/MasterData/WALs/da1c6afcd1f9,42151,1731595912734/da1c6afcd1f9%2C42151%2C1731595912734.1731595914006 2024-11-14T14:51:54,166 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33623:33623),(127.0.0.1/127.0.0.1:33515:33515),(127.0.0.1/127.0.0.1:34777:34777)] 2024-11-14T14:51:54,166 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-14T14:51:54,167 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T14:51:54,170 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-14T14:51:54,171 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-14T14:51:54,209 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-14T14:51:54,236 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-14T14:51:54,240 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T14:51:54,244 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T14:51:54,244 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-14T14:51:54,248 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-14T14:51:54,248 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T14:51:54,249 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T14:51:54,250 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-14T14:51:54,252 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-14T14:51:54,252 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T14:51:54,253 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T14:51:54,254 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-14T14:51:54,256 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-14T14:51:54,256 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T14:51:54,257 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T14:51:54,257 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-14T14:51:54,262 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34445/user/jenkins/test-data/35161b07-018a-0344-8c43-1519f3812567/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-14T14:51:54,263 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34445/user/jenkins/test-data/35161b07-018a-0344-8c43-1519f3812567/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-14T14:51:54,268 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-14T14:51:54,269 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-14T14:51:54,273 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-14T14:51:54,277 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-14T14:51:54,281 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34445/user/jenkins/test-data/35161b07-018a-0344-8c43-1519f3812567/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-14T14:51:54,283 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63829801, jitterRate=-0.04886184632778168}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-14T14:51:54,288 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731595914184Initializing all the Stores at 1731595914186 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731595914186Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731595914187 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731595914188 (+1 ms)Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731595914188Cleaning up temporary data from old regions at 1731595914269 (+81 ms)Region opened successfully at 1731595914288 (+19 ms) 2024-11-14T14:51:54,289 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-14T14:51:54,326 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@28ff8426, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=da1c6afcd1f9/172.17.0.2:0 2024-11-14T14:51:54,360 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-14T14:51:54,372 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-14T14:51:54,372 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-14T14:51:54,375 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-14T14:51:54,377 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-14T14:51:54,383 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-11-14T14:51:54,383 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-14T14:51:54,415 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-14T14:51:54,427 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42151-0x101a74438830000, quorum=127.0.0.1:51231, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-14T14:51:54,429 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-14T14:51:54,432 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-14T14:51:54,434 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42151-0x101a74438830000, quorum=127.0.0.1:51231, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-14T14:51:54,436 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-14T14:51:54,438 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-14T14:51:54,442 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42151-0x101a74438830000, quorum=127.0.0.1:51231, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-14T14:51:54,443 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-14T14:51:54,444 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42151-0x101a74438830000, quorum=127.0.0.1:51231, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-14T14:51:54,446 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-14T14:51:54,466 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42151-0x101a74438830000, quorum=127.0.0.1:51231, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-14T14:51:54,467 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-14T14:51:54,472 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46007-0x101a74438830003, quorum=127.0.0.1:51231, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-14T14:51:54,472 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37695-0x101a74438830001, quorum=127.0.0.1:51231, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-14T14:51:54,472 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46007-0x101a74438830003, quorum=127.0.0.1:51231, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T14:51:54,472 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42151-0x101a74438830000, quorum=127.0.0.1:51231, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-14T14:51:54,472 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34155-0x101a74438830002, quorum=127.0.0.1:51231, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-14T14:51:54,472 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37695-0x101a74438830001, quorum=127.0.0.1:51231, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T14:51:54,472 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42151-0x101a74438830000, quorum=127.0.0.1:51231, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T14:51:54,472 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34155-0x101a74438830002, quorum=127.0.0.1:51231, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T14:51:54,475 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=da1c6afcd1f9,42151,1731595912734, sessionid=0x101a74438830000, setting cluster-up flag (Was=false) 2024-11-14T14:51:54,489 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34155-0x101a74438830002, quorum=127.0.0.1:51231, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T14:51:54,489 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46007-0x101a74438830003, quorum=127.0.0.1:51231, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T14:51:54,489 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42151-0x101a74438830000, quorum=127.0.0.1:51231, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T14:51:54,489 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37695-0x101a74438830001, quorum=127.0.0.1:51231, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T14:51:54,495 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-14T14:51:54,497 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=da1c6afcd1f9,42151,1731595912734 2024-11-14T14:51:54,502 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42151-0x101a74438830000, quorum=127.0.0.1:51231, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T14:51:54,502 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34155-0x101a74438830002, quorum=127.0.0.1:51231, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T14:51:54,502 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37695-0x101a74438830001, quorum=127.0.0.1:51231, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T14:51:54,502 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46007-0x101a74438830003, quorum=127.0.0.1:51231, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T14:51:54,508 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-14T14:51:54,510 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=da1c6afcd1f9,42151,1731595912734 2024-11-14T14:51:54,517 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:34445/user/jenkins/test-data/35161b07-018a-0344-8c43-1519f3812567/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-14T14:51:54,519 INFO [RS:0;da1c6afcd1f9:37695 {}] regionserver.HRegionServer(746): ClusterId : 72d32361-5e69-4ee6-9ad5-db998b6b568a 2024-11-14T14:51:54,519 INFO [RS:2;da1c6afcd1f9:46007 {}] regionserver.HRegionServer(746): ClusterId : 72d32361-5e69-4ee6-9ad5-db998b6b568a 2024-11-14T14:51:54,519 INFO [RS:1;da1c6afcd1f9:34155 {}] regionserver.HRegionServer(746): ClusterId : 72d32361-5e69-4ee6-9ad5-db998b6b568a 2024-11-14T14:51:54,522 DEBUG [RS:1;da1c6afcd1f9:34155 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-14T14:51:54,522 DEBUG [RS:2;da1c6afcd1f9:46007 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-14T14:51:54,522 DEBUG [RS:0;da1c6afcd1f9:37695 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-14T14:51:54,528 DEBUG [RS:2;da1c6afcd1f9:46007 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-14T14:51:54,528 DEBUG [RS:0;da1c6afcd1f9:37695 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-14T14:51:54,528 DEBUG [RS:1;da1c6afcd1f9:34155 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-14T14:51:54,528 DEBUG [RS:2;da1c6afcd1f9:46007 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-14T14:51:54,528 DEBUG [RS:0;da1c6afcd1f9:37695 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-14T14:51:54,528 DEBUG [RS:1;da1c6afcd1f9:34155 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-14T14:51:54,531 DEBUG [RS:0;da1c6afcd1f9:37695 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-14T14:51:54,531 DEBUG [RS:2;da1c6afcd1f9:46007 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-14T14:51:54,531 DEBUG [RS:1;da1c6afcd1f9:34155 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-14T14:51:54,532 DEBUG [RS:0;da1c6afcd1f9:37695 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5736518b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=da1c6afcd1f9/172.17.0.2:0 2024-11-14T14:51:54,532 DEBUG [RS:2;da1c6afcd1f9:46007 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@299cba62, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=da1c6afcd1f9/172.17.0.2:0 2024-11-14T14:51:54,532 DEBUG [RS:1;da1c6afcd1f9:34155 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@215f6b6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=da1c6afcd1f9/172.17.0.2:0 2024-11-14T14:51:54,548 DEBUG [RS:0;da1c6afcd1f9:37695 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;da1c6afcd1f9:37695 2024-11-14T14:51:54,552 INFO [RS:0;da1c6afcd1f9:37695 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-14T14:51:54,552 INFO [RS:0;da1c6afcd1f9:37695 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-14T14:51:54,552 DEBUG [RS:0;da1c6afcd1f9:37695 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-14T14:51:54,552 DEBUG [RS:2;da1c6afcd1f9:46007 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:2;da1c6afcd1f9:46007 2024-11-14T14:51:54,553 INFO [RS:2;da1c6afcd1f9:46007 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-14T14:51:54,553 INFO [RS:2;da1c6afcd1f9:46007 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-14T14:51:54,553 DEBUG [RS:2;da1c6afcd1f9:46007 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-14T14:51:54,554 DEBUG [RS:1;da1c6afcd1f9:34155 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;da1c6afcd1f9:34155 2024-11-14T14:51:54,554 INFO [RS:1;da1c6afcd1f9:34155 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-14T14:51:54,554 INFO [RS:1;da1c6afcd1f9:34155 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-14T14:51:54,554 DEBUG [RS:1;da1c6afcd1f9:34155 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-14T14:51:54,556 INFO [RS:2;da1c6afcd1f9:46007 {}] regionserver.HRegionServer(2659): reportForDuty to master=da1c6afcd1f9,42151,1731595912734 with port=46007, startcode=1731595913586 2024-11-14T14:51:54,556 INFO [RS:1;da1c6afcd1f9:34155 {}] regionserver.HRegionServer(2659): reportForDuty to master=da1c6afcd1f9,42151,1731595912734 with port=34155, startcode=1731595913533 2024-11-14T14:51:54,556 INFO [RS:0;da1c6afcd1f9:37695 {}] regionserver.HRegionServer(2659): reportForDuty to master=da1c6afcd1f9,42151,1731595912734 with port=37695, startcode=1731595913427 2024-11-14T14:51:54,570 DEBUG [RS:1;da1c6afcd1f9:34155 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-14T14:51:54,570 DEBUG [RS:2;da1c6afcd1f9:46007 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-14T14:51:54,570 DEBUG [RS:0;da1c6afcd1f9:37695 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-14T14:51:54,614 INFO [HMaster-EventLoopGroup-2-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53003, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.2 (auth:SIMPLE), service=RegionServerStatusService 2024-11-14T14:51:54,614 INFO [HMaster-EventLoopGroup-2-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38589, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-14T14:51:54,614 INFO [HMaster-EventLoopGroup-2-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39267, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.1 (auth:SIMPLE), service=RegionServerStatusService 2024-11-14T14:51:54,619 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-14T14:51:54,620 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42151 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T14:51:54,626 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42151 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T14:51:54,627 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42151 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T14:51:54,633 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-14T14:51:54,641 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-14T14:51:54,646 DEBUG [RS:2;da1c6afcd1f9:46007 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-11-14T14:51:54,646 DEBUG [RS:0;da1c6afcd1f9:37695 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-11-14T14:51:54,646 DEBUG [RS:1;da1c6afcd1f9:34155 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-11-14T14:51:54,647 WARN [RS:0;da1c6afcd1f9:37695 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-11-14T14:51:54,647 WARN [RS:2;da1c6afcd1f9:46007 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-11-14T14:51:54,647 WARN [RS:1;da1c6afcd1f9:34155 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-11-14T14:51:54,648 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: da1c6afcd1f9,42151,1731595912734 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-14T14:51:54,656 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/da1c6afcd1f9:0, corePoolSize=5, maxPoolSize=5 2024-11-14T14:51:54,656 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/da1c6afcd1f9:0, corePoolSize=5, maxPoolSize=5 2024-11-14T14:51:54,657 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/da1c6afcd1f9:0, corePoolSize=5, maxPoolSize=5 2024-11-14T14:51:54,657 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/da1c6afcd1f9:0, corePoolSize=5, maxPoolSize=5 2024-11-14T14:51:54,657 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/da1c6afcd1f9:0, corePoolSize=10, maxPoolSize=10 2024-11-14T14:51:54,657 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/da1c6afcd1f9:0, corePoolSize=1, maxPoolSize=1 2024-11-14T14:51:54,657 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/da1c6afcd1f9:0, corePoolSize=2, maxPoolSize=2 2024-11-14T14:51:54,657 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/da1c6afcd1f9:0, corePoolSize=1, maxPoolSize=1 2024-11-14T14:51:54,658 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731595944658 2024-11-14T14:51:54,660 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-14T14:51:54,661 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-14T14:51:54,665 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-14T14:51:54,665 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-14T14:51:54,665 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-14T14:51:54,666 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-14T14:51:54,666 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-14T14:51:54,666 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-14T14:51:54,667 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-14T14:51:54,670 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-14T14:51:54,671 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-14T14:51:54,672 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-14T14:51:54,673 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T14:51:54,673 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-14T14:51:54,674 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-14T14:51:54,675 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-14T14:51:54,677 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/da1c6afcd1f9:0:becomeActiveMaster-HFileCleaner.large.0-1731595914676,5,FailOnTimeoutGroup] 2024-11-14T14:51:54,678 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/da1c6afcd1f9:0:becomeActiveMaster-HFileCleaner.small.0-1731595914677,5,FailOnTimeoutGroup] 2024-11-14T14:51:54,678 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-14T14:51:54,678 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-14T14:51:54,679 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-14T14:51:54,680 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-14T14:51:54,710 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42819 is added to blk_1073741831_1007 (size=1321) 2024-11-14T14:51:54,711 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36189 is added to blk_1073741831_1007 (size=1321) 2024-11-14T14:51:54,712 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44549 is added to blk_1073741831_1007 (size=1321) 2024-11-14T14:51:54,714 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:34445/user/jenkins/test-data/35161b07-018a-0344-8c43-1519f3812567/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-14T14:51:54,714 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:34445/user/jenkins/test-data/35161b07-018a-0344-8c43-1519f3812567 2024-11-14T14:51:54,727 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42819 is added to blk_1073741832_1008 (size=32) 2024-11-14T14:51:54,727 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44549 is added to blk_1073741832_1008 (size=32) 2024-11-14T14:51:54,728 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36189 is added to blk_1073741832_1008 (size=32) 2024-11-14T14:51:54,730 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T14:51:54,732 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-14T14:51:54,735 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-14T14:51:54,735 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T14:51:54,736 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T14:51:54,736 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-14T14:51:54,738 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-14T14:51:54,738 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T14:51:54,739 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T14:51:54,739 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-14T14:51:54,741 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-14T14:51:54,742 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T14:51:54,742 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T14:51:54,743 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-14T14:51:54,745 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-14T14:51:54,745 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T14:51:54,746 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T14:51:54,746 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-14T14:51:54,748 INFO [RS:1;da1c6afcd1f9:34155 {}] regionserver.HRegionServer(2659): reportForDuty to master=da1c6afcd1f9,42151,1731595912734 with port=34155, startcode=1731595913533 2024-11-14T14:51:54,748 INFO [RS:0;da1c6afcd1f9:37695 {}] regionserver.HRegionServer(2659): reportForDuty to master=da1c6afcd1f9,42151,1731595912734 with port=37695, startcode=1731595913427 2024-11-14T14:51:54,748 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34445/user/jenkins/test-data/35161b07-018a-0344-8c43-1519f3812567/data/hbase/meta/1588230740 2024-11-14T14:51:54,748 INFO [RS:2;da1c6afcd1f9:46007 {}] regionserver.HRegionServer(2659): reportForDuty to master=da1c6afcd1f9,42151,1731595912734 with port=46007, startcode=1731595913586 2024-11-14T14:51:54,750 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34445/user/jenkins/test-data/35161b07-018a-0344-8c43-1519f3812567/data/hbase/meta/1588230740 2024-11-14T14:51:54,750 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42151 {}] master.ServerManager(363): Checking decommissioned status of RegionServer da1c6afcd1f9,37695,1731595913427 2024-11-14T14:51:54,753 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-14T14:51:54,753 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-14T14:51:54,753 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42151 {}] master.ServerManager(517): Registering regionserver=da1c6afcd1f9,37695,1731595913427 2024-11-14T14:51:54,754 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-14T14:51:54,756 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-14T14:51:54,761 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34445/user/jenkins/test-data/35161b07-018a-0344-8c43-1519f3812567/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-14T14:51:54,762 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64673001, jitterRate=-0.036297187209129333}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-14T14:51:54,763 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42151 {}] master.ServerManager(363): Checking decommissioned status of RegionServer da1c6afcd1f9,34155,1731595913533 2024-11-14T14:51:54,763 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42151 {}] master.ServerManager(517): Registering regionserver=da1c6afcd1f9,34155,1731595913533 2024-11-14T14:51:54,764 DEBUG [RS:0;da1c6afcd1f9:37695 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:34445/user/jenkins/test-data/35161b07-018a-0344-8c43-1519f3812567 2024-11-14T14:51:54,764 DEBUG [RS:0;da1c6afcd1f9:37695 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:34445 2024-11-14T14:51:54,764 DEBUG [RS:0;da1c6afcd1f9:37695 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-14T14:51:54,766 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731595914730Initializing all the Stores at 1731595914732 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731595914732Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731595914732Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731595914732Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731595914732Cleaning up temporary data from old regions at 1731595914753 (+21 ms)Region opened successfully at 1731595914766 (+13 ms) 2024-11-14T14:51:54,766 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-14T14:51:54,766 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-14T14:51:54,766 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-14T14:51:54,767 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-14T14:51:54,767 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-14T14:51:54,768 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-14T14:51:54,768 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731595914766Disabling compacts and flushes for region at 1731595914766Disabling writes for close at 1731595914767 (+1 ms)Writing region close event to WAL at 1731595914767Closed at 1731595914768 (+1 ms) 2024-11-14T14:51:54,769 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42151 {}] master.ServerManager(363): Checking decommissioned status of RegionServer da1c6afcd1f9,46007,1731595913586 2024-11-14T14:51:54,769 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42151 {}] master.ServerManager(517): Registering regionserver=da1c6afcd1f9,46007,1731595913586 2024-11-14T14:51:54,769 DEBUG [RS:1;da1c6afcd1f9:34155 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:34445/user/jenkins/test-data/35161b07-018a-0344-8c43-1519f3812567 2024-11-14T14:51:54,769 DEBUG [RS:1;da1c6afcd1f9:34155 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:34445 2024-11-14T14:51:54,769 DEBUG [RS:1;da1c6afcd1f9:34155 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-14T14:51:54,770 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42151-0x101a74438830000, quorum=127.0.0.1:51231, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-14T14:51:54,772 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-14T14:51:54,772 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-14T14:51:54,773 DEBUG [RS:2;da1c6afcd1f9:46007 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:34445/user/jenkins/test-data/35161b07-018a-0344-8c43-1519f3812567 2024-11-14T14:51:54,773 DEBUG [RS:2;da1c6afcd1f9:46007 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:34445 2024-11-14T14:51:54,773 DEBUG [RS:2;da1c6afcd1f9:46007 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-14T14:51:54,774 DEBUG [RS:0;da1c6afcd1f9:37695 {}] zookeeper.ZKUtil(111): regionserver:37695-0x101a74438830001, quorum=127.0.0.1:51231, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/da1c6afcd1f9,37695,1731595913427 2024-11-14T14:51:54,775 WARN [RS:0;da1c6afcd1f9:37695 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-14T14:51:54,775 INFO [RS:0;da1c6afcd1f9:37695 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-14T14:51:54,775 DEBUG [RS:0;da1c6afcd1f9:37695 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:34445/user/jenkins/test-data/35161b07-018a-0344-8c43-1519f3812567/WALs/da1c6afcd1f9,37695,1731595913427 2024-11-14T14:51:54,775 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42151-0x101a74438830000, quorum=127.0.0.1:51231, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-14T14:51:54,776 DEBUG [RS:1;da1c6afcd1f9:34155 {}] zookeeper.ZKUtil(111): regionserver:34155-0x101a74438830002, quorum=127.0.0.1:51231, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/da1c6afcd1f9,34155,1731595913533 2024-11-14T14:51:54,776 WARN [RS:1;da1c6afcd1f9:34155 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-14T14:51:54,776 INFO [RS:1;da1c6afcd1f9:34155 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-14T14:51:54,776 DEBUG [RS:1;da1c6afcd1f9:34155 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:34445/user/jenkins/test-data/35161b07-018a-0344-8c43-1519f3812567/WALs/da1c6afcd1f9,34155,1731595913533 2024-11-14T14:51:54,776 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [da1c6afcd1f9,37695,1731595913427] 2024-11-14T14:51:54,776 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [da1c6afcd1f9,34155,1731595913533] 2024-11-14T14:51:54,778 DEBUG [RS:2;da1c6afcd1f9:46007 {}] zookeeper.ZKUtil(111): regionserver:46007-0x101a74438830003, quorum=127.0.0.1:51231, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/da1c6afcd1f9,46007,1731595913586 2024-11-14T14:51:54,778 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [da1c6afcd1f9,46007,1731595913586] 2024-11-14T14:51:54,778 WARN [RS:2;da1c6afcd1f9:46007 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-14T14:51:54,778 INFO [RS:2;da1c6afcd1f9:46007 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-14T14:51:54,778 DEBUG [RS:2;da1c6afcd1f9:46007 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:34445/user/jenkins/test-data/35161b07-018a-0344-8c43-1519f3812567/WALs/da1c6afcd1f9,46007,1731595913586 2024-11-14T14:51:54,782 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-14T14:51:54,793 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-14T14:51:54,796 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-14T14:51:54,808 INFO [RS:0;da1c6afcd1f9:37695 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-14T14:51:54,808 INFO [RS:2;da1c6afcd1f9:46007 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-14T14:51:54,808 INFO [RS:1;da1c6afcd1f9:34155 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-14T14:51:54,826 INFO [RS:1;da1c6afcd1f9:34155 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-14T14:51:54,826 INFO [RS:0;da1c6afcd1f9:37695 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-14T14:51:54,826 INFO [RS:2;da1c6afcd1f9:46007 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-14T14:51:54,832 INFO [RS:0;da1c6afcd1f9:37695 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-14T14:51:54,832 INFO [RS:1;da1c6afcd1f9:34155 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-14T14:51:54,832 INFO [RS:1;da1c6afcd1f9:34155 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-14T14:51:54,832 INFO [RS:0;da1c6afcd1f9:37695 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-14T14:51:54,832 INFO [RS:2;da1c6afcd1f9:46007 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-14T14:51:54,832 INFO [RS:2;da1c6afcd1f9:46007 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-14T14:51:54,833 INFO [RS:1;da1c6afcd1f9:34155 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-14T14:51:54,833 INFO [RS:2;da1c6afcd1f9:46007 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-14T14:51:54,833 INFO [RS:0;da1c6afcd1f9:37695 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-14T14:51:54,839 INFO [RS:0;da1c6afcd1f9:37695 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-14T14:51:54,839 INFO [RS:1;da1c6afcd1f9:34155 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-14T14:51:54,839 INFO [RS:2;da1c6afcd1f9:46007 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-14T14:51:54,841 INFO [RS:2;da1c6afcd1f9:46007 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-14T14:51:54,841 INFO [RS:1;da1c6afcd1f9:34155 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-14T14:51:54,841 INFO [RS:0;da1c6afcd1f9:37695 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-14T14:51:54,841 DEBUG [RS:2;da1c6afcd1f9:46007 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/da1c6afcd1f9:0, corePoolSize=1, maxPoolSize=1 2024-11-14T14:51:54,841 DEBUG [RS:1;da1c6afcd1f9:34155 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/da1c6afcd1f9:0, corePoolSize=1, maxPoolSize=1 2024-11-14T14:51:54,841 DEBUG [RS:0;da1c6afcd1f9:37695 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/da1c6afcd1f9:0, corePoolSize=1, maxPoolSize=1 2024-11-14T14:51:54,841 DEBUG [RS:2;da1c6afcd1f9:46007 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/da1c6afcd1f9:0, corePoolSize=1, maxPoolSize=1 2024-11-14T14:51:54,841 DEBUG [RS:1;da1c6afcd1f9:34155 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/da1c6afcd1f9:0, corePoolSize=1, maxPoolSize=1 2024-11-14T14:51:54,841 DEBUG [RS:0;da1c6afcd1f9:37695 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/da1c6afcd1f9:0, corePoolSize=1, maxPoolSize=1 2024-11-14T14:51:54,841 DEBUG [RS:1;da1c6afcd1f9:34155 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/da1c6afcd1f9:0, corePoolSize=1, maxPoolSize=1 2024-11-14T14:51:54,841 DEBUG [RS:2;da1c6afcd1f9:46007 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/da1c6afcd1f9:0, corePoolSize=1, maxPoolSize=1 2024-11-14T14:51:54,841 DEBUG [RS:0;da1c6afcd1f9:37695 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/da1c6afcd1f9:0, corePoolSize=1, maxPoolSize=1 2024-11-14T14:51:54,842 DEBUG [RS:2;da1c6afcd1f9:46007 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0, corePoolSize=1, maxPoolSize=1 2024-11-14T14:51:54,842 DEBUG [RS:1;da1c6afcd1f9:34155 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0, corePoolSize=1, maxPoolSize=1 2024-11-14T14:51:54,842 DEBUG [RS:0;da1c6afcd1f9:37695 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0, corePoolSize=1, maxPoolSize=1 2024-11-14T14:51:54,842 DEBUG [RS:1;da1c6afcd1f9:34155 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/da1c6afcd1f9:0, corePoolSize=1, maxPoolSize=1 2024-11-14T14:51:54,842 DEBUG [RS:2;da1c6afcd1f9:46007 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/da1c6afcd1f9:0, corePoolSize=1, maxPoolSize=1 2024-11-14T14:51:54,842 DEBUG [RS:0;da1c6afcd1f9:37695 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/da1c6afcd1f9:0, corePoolSize=1, maxPoolSize=1 2024-11-14T14:51:54,842 DEBUG [RS:1;da1c6afcd1f9:34155 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/da1c6afcd1f9:0, corePoolSize=2, maxPoolSize=2 2024-11-14T14:51:54,842 DEBUG [RS:2;da1c6afcd1f9:46007 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/da1c6afcd1f9:0, corePoolSize=2, maxPoolSize=2 2024-11-14T14:51:54,842 DEBUG [RS:0;da1c6afcd1f9:37695 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/da1c6afcd1f9:0, corePoolSize=2, maxPoolSize=2 2024-11-14T14:51:54,842 DEBUG [RS:1;da1c6afcd1f9:34155 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/da1c6afcd1f9:0, corePoolSize=1, maxPoolSize=1 2024-11-14T14:51:54,842 DEBUG [RS:2;da1c6afcd1f9:46007 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/da1c6afcd1f9:0, corePoolSize=1, maxPoolSize=1 2024-11-14T14:51:54,842 DEBUG [RS:0;da1c6afcd1f9:37695 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/da1c6afcd1f9:0, corePoolSize=1, maxPoolSize=1 2024-11-14T14:51:54,842 DEBUG [RS:2;da1c6afcd1f9:46007 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/da1c6afcd1f9:0, corePoolSize=1, maxPoolSize=1 2024-11-14T14:51:54,842 DEBUG [RS:1;da1c6afcd1f9:34155 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/da1c6afcd1f9:0, corePoolSize=1, maxPoolSize=1 2024-11-14T14:51:54,843 DEBUG [RS:0;da1c6afcd1f9:37695 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/da1c6afcd1f9:0, corePoolSize=1, maxPoolSize=1 2024-11-14T14:51:54,843 DEBUG [RS:1;da1c6afcd1f9:34155 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/da1c6afcd1f9:0, corePoolSize=1, maxPoolSize=1 2024-11-14T14:51:54,843 DEBUG [RS:2;da1c6afcd1f9:46007 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/da1c6afcd1f9:0, corePoolSize=1, maxPoolSize=1 2024-11-14T14:51:54,843 DEBUG [RS:0;da1c6afcd1f9:37695 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/da1c6afcd1f9:0, corePoolSize=1, maxPoolSize=1 2024-11-14T14:51:54,843 DEBUG [RS:1;da1c6afcd1f9:34155 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/da1c6afcd1f9:0, corePoolSize=1, maxPoolSize=1 2024-11-14T14:51:54,843 DEBUG [RS:2;da1c6afcd1f9:46007 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/da1c6afcd1f9:0, corePoolSize=1, maxPoolSize=1 2024-11-14T14:51:54,843 DEBUG [RS:1;da1c6afcd1f9:34155 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/da1c6afcd1f9:0, corePoolSize=1, maxPoolSize=1 2024-11-14T14:51:54,843 DEBUG [RS:2;da1c6afcd1f9:46007 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/da1c6afcd1f9:0, corePoolSize=1, maxPoolSize=1 2024-11-14T14:51:54,843 DEBUG [RS:0;da1c6afcd1f9:37695 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/da1c6afcd1f9:0, corePoolSize=1, maxPoolSize=1 2024-11-14T14:51:54,843 DEBUG [RS:2;da1c6afcd1f9:46007 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/da1c6afcd1f9:0, corePoolSize=1, maxPoolSize=1 2024-11-14T14:51:54,843 DEBUG [RS:1;da1c6afcd1f9:34155 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/da1c6afcd1f9:0, corePoolSize=1, maxPoolSize=1 2024-11-14T14:51:54,843 DEBUG [RS:0;da1c6afcd1f9:37695 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/da1c6afcd1f9:0, corePoolSize=1, maxPoolSize=1 2024-11-14T14:51:54,843 DEBUG [RS:2;da1c6afcd1f9:46007 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/da1c6afcd1f9:0, corePoolSize=3, maxPoolSize=3 2024-11-14T14:51:54,843 DEBUG [RS:1;da1c6afcd1f9:34155 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/da1c6afcd1f9:0, corePoolSize=3, maxPoolSize=3 2024-11-14T14:51:54,843 DEBUG [RS:0;da1c6afcd1f9:37695 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/da1c6afcd1f9:0, corePoolSize=1, maxPoolSize=1 2024-11-14T14:51:54,843 DEBUG [RS:1;da1c6afcd1f9:34155 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/da1c6afcd1f9:0, corePoolSize=3, maxPoolSize=3 2024-11-14T14:51:54,843 DEBUG [RS:2;da1c6afcd1f9:46007 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/da1c6afcd1f9:0, corePoolSize=3, maxPoolSize=3 2024-11-14T14:51:54,844 DEBUG [RS:0;da1c6afcd1f9:37695 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/da1c6afcd1f9:0, corePoolSize=3, maxPoolSize=3 2024-11-14T14:51:54,844 DEBUG [RS:0;da1c6afcd1f9:37695 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/da1c6afcd1f9:0, corePoolSize=3, maxPoolSize=3 2024-11-14T14:51:54,851 INFO [RS:1;da1c6afcd1f9:34155 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-14T14:51:54,851 INFO [RS:1;da1c6afcd1f9:34155 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-14T14:51:54,851 INFO [RS:1;da1c6afcd1f9:34155 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-14T14:51:54,851 INFO [RS:1;da1c6afcd1f9:34155 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-14T14:51:54,851 INFO [RS:1;da1c6afcd1f9:34155 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-14T14:51:54,851 INFO [RS:1;da1c6afcd1f9:34155 {}] hbase.ChoreService(168): Chore ScheduledChore name=da1c6afcd1f9,34155,1731595913533-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-14T14:51:54,854 INFO [RS:2;da1c6afcd1f9:46007 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-14T14:51:54,854 INFO [RS:2;da1c6afcd1f9:46007 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-14T14:51:54,854 INFO [RS:2;da1c6afcd1f9:46007 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-14T14:51:54,854 INFO [RS:2;da1c6afcd1f9:46007 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-14T14:51:54,854 INFO [RS:2;da1c6afcd1f9:46007 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-14T14:51:54,854 INFO [RS:2;da1c6afcd1f9:46007 {}] hbase.ChoreService(168): Chore ScheduledChore name=da1c6afcd1f9,46007,1731595913586-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-14T14:51:54,856 INFO [RS:0;da1c6afcd1f9:37695 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-14T14:51:54,856 INFO [RS:0;da1c6afcd1f9:37695 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-14T14:51:54,856 INFO [RS:0;da1c6afcd1f9:37695 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-14T14:51:54,856 INFO [RS:0;da1c6afcd1f9:37695 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-14T14:51:54,856 INFO [RS:0;da1c6afcd1f9:37695 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-14T14:51:54,856 INFO [RS:0;da1c6afcd1f9:37695 {}] hbase.ChoreService(168): Chore ScheduledChore name=da1c6afcd1f9,37695,1731595913427-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-14T14:51:54,876 INFO [RS:0;da1c6afcd1f9:37695 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-14T14:51:54,878 INFO [RS:0;da1c6afcd1f9:37695 {}] hbase.ChoreService(168): Chore ScheduledChore name=da1c6afcd1f9,37695,1731595913427-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-14T14:51:54,879 INFO [RS:0;da1c6afcd1f9:37695 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-14T14:51:54,879 INFO [RS:0;da1c6afcd1f9:37695 {}] regionserver.Replication(171): da1c6afcd1f9,37695,1731595913427 started 2024-11-14T14:51:54,881 INFO [RS:2;da1c6afcd1f9:46007 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-14T14:51:54,881 INFO [RS:1;da1c6afcd1f9:34155 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-14T14:51:54,881 INFO [RS:1;da1c6afcd1f9:34155 {}] hbase.ChoreService(168): Chore ScheduledChore name=da1c6afcd1f9,34155,1731595913533-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-14T14:51:54,881 INFO [RS:2;da1c6afcd1f9:46007 {}] hbase.ChoreService(168): Chore ScheduledChore name=da1c6afcd1f9,46007,1731595913586-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-14T14:51:54,881 INFO [RS:2;da1c6afcd1f9:46007 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-14T14:51:54,881 INFO [RS:1;da1c6afcd1f9:34155 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-14T14:51:54,882 INFO [RS:2;da1c6afcd1f9:46007 {}] regionserver.Replication(171): da1c6afcd1f9,46007,1731595913586 started 2024-11-14T14:51:54,882 INFO [RS:1;da1c6afcd1f9:34155 {}] regionserver.Replication(171): da1c6afcd1f9,34155,1731595913533 started 2024-11-14T14:51:54,898 INFO [RS:0;da1c6afcd1f9:37695 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-14T14:51:54,899 INFO [RS:0;da1c6afcd1f9:37695 {}] regionserver.HRegionServer(1482): Serving as da1c6afcd1f9,37695,1731595913427, RpcServer on da1c6afcd1f9/172.17.0.2:37695, sessionid=0x101a74438830001 2024-11-14T14:51:54,900 DEBUG [RS:0;da1c6afcd1f9:37695 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-14T14:51:54,900 DEBUG [RS:0;da1c6afcd1f9:37695 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager da1c6afcd1f9,37695,1731595913427 2024-11-14T14:51:54,900 DEBUG [RS:0;da1c6afcd1f9:37695 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'da1c6afcd1f9,37695,1731595913427' 2024-11-14T14:51:54,900 DEBUG [RS:0;da1c6afcd1f9:37695 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-14T14:51:54,901 DEBUG [RS:0;da1c6afcd1f9:37695 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-14T14:51:54,902 DEBUG [RS:0;da1c6afcd1f9:37695 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-14T14:51:54,902 DEBUG [RS:0;da1c6afcd1f9:37695 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-14T14:51:54,902 DEBUG [RS:0;da1c6afcd1f9:37695 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager da1c6afcd1f9,37695,1731595913427 2024-11-14T14:51:54,902 DEBUG [RS:0;da1c6afcd1f9:37695 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'da1c6afcd1f9,37695,1731595913427' 2024-11-14T14:51:54,902 DEBUG [RS:0;da1c6afcd1f9:37695 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-14T14:51:54,903 DEBUG [RS:0;da1c6afcd1f9:37695 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-14T14:51:54,904 DEBUG [RS:0;da1c6afcd1f9:37695 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-14T14:51:54,904 INFO [RS:0;da1c6afcd1f9:37695 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-14T14:51:54,904 INFO [RS:0;da1c6afcd1f9:37695 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-14T14:51:54,906 INFO [RS:1;da1c6afcd1f9:34155 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-14T14:51:54,906 INFO [RS:2;da1c6afcd1f9:46007 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-14T14:51:54,906 INFO [RS:1;da1c6afcd1f9:34155 {}] regionserver.HRegionServer(1482): Serving as da1c6afcd1f9,34155,1731595913533, RpcServer on da1c6afcd1f9/172.17.0.2:34155, sessionid=0x101a74438830002 2024-11-14T14:51:54,906 INFO [RS:2;da1c6afcd1f9:46007 {}] regionserver.HRegionServer(1482): Serving as da1c6afcd1f9,46007,1731595913586, RpcServer on da1c6afcd1f9/172.17.0.2:46007, sessionid=0x101a74438830003 2024-11-14T14:51:54,906 DEBUG [RS:1;da1c6afcd1f9:34155 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-14T14:51:54,906 DEBUG [RS:2;da1c6afcd1f9:46007 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-14T14:51:54,906 DEBUG [RS:2;da1c6afcd1f9:46007 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager da1c6afcd1f9,46007,1731595913586 2024-11-14T14:51:54,906 DEBUG [RS:1;da1c6afcd1f9:34155 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager da1c6afcd1f9,34155,1731595913533 2024-11-14T14:51:54,907 DEBUG [RS:2;da1c6afcd1f9:46007 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'da1c6afcd1f9,46007,1731595913586' 2024-11-14T14:51:54,907 DEBUG [RS:1;da1c6afcd1f9:34155 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'da1c6afcd1f9,34155,1731595913533' 2024-11-14T14:51:54,907 DEBUG [RS:2;da1c6afcd1f9:46007 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-14T14:51:54,907 DEBUG [RS:1;da1c6afcd1f9:34155 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-14T14:51:54,907 DEBUG [RS:2;da1c6afcd1f9:46007 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-14T14:51:54,907 DEBUG [RS:1;da1c6afcd1f9:34155 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-14T14:51:54,908 DEBUG [RS:2;da1c6afcd1f9:46007 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-14T14:51:54,908 DEBUG [RS:1;da1c6afcd1f9:34155 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-14T14:51:54,908 DEBUG [RS:2;da1c6afcd1f9:46007 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-14T14:51:54,908 DEBUG [RS:1;da1c6afcd1f9:34155 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-14T14:51:54,908 DEBUG [RS:2;da1c6afcd1f9:46007 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager da1c6afcd1f9,46007,1731595913586 2024-11-14T14:51:54,908 DEBUG [RS:1;da1c6afcd1f9:34155 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager da1c6afcd1f9,34155,1731595913533 2024-11-14T14:51:54,908 DEBUG [RS:2;da1c6afcd1f9:46007 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'da1c6afcd1f9,46007,1731595913586' 2024-11-14T14:51:54,908 DEBUG [RS:1;da1c6afcd1f9:34155 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'da1c6afcd1f9,34155,1731595913533' 2024-11-14T14:51:54,908 DEBUG [RS:2;da1c6afcd1f9:46007 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-14T14:51:54,908 DEBUG [RS:1;da1c6afcd1f9:34155 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-14T14:51:54,909 DEBUG [RS:1;da1c6afcd1f9:34155 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-14T14:51:54,909 DEBUG [RS:2;da1c6afcd1f9:46007 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-14T14:51:54,909 DEBUG [RS:2;da1c6afcd1f9:46007 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-14T14:51:54,909 DEBUG [RS:1;da1c6afcd1f9:34155 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-14T14:51:54,910 INFO [RS:1;da1c6afcd1f9:34155 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-14T14:51:54,910 INFO [RS:2;da1c6afcd1f9:46007 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-14T14:51:54,910 INFO [RS:1;da1c6afcd1f9:34155 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-14T14:51:54,910 INFO [RS:2;da1c6afcd1f9:46007 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-14T14:51:54,947 WARN [da1c6afcd1f9:42151 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-14T14:51:55,009 INFO [RS:0;da1c6afcd1f9:37695 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-14T14:51:55,011 INFO [RS:1;da1c6afcd1f9:34155 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-14T14:51:55,011 INFO [RS:2;da1c6afcd1f9:46007 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-14T14:51:55,013 INFO [RS:0;da1c6afcd1f9:37695 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=da1c6afcd1f9%2C37695%2C1731595913427, suffix=, logDir=hdfs://localhost:34445/user/jenkins/test-data/35161b07-018a-0344-8c43-1519f3812567/WALs/da1c6afcd1f9,37695,1731595913427, archiveDir=hdfs://localhost:34445/user/jenkins/test-data/35161b07-018a-0344-8c43-1519f3812567/oldWALs, maxLogs=32 2024-11-14T14:51:55,014 INFO [RS:1;da1c6afcd1f9:34155 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=da1c6afcd1f9%2C34155%2C1731595913533, suffix=, logDir=hdfs://localhost:34445/user/jenkins/test-data/35161b07-018a-0344-8c43-1519f3812567/WALs/da1c6afcd1f9,34155,1731595913533, archiveDir=hdfs://localhost:34445/user/jenkins/test-data/35161b07-018a-0344-8c43-1519f3812567/oldWALs, maxLogs=32 2024-11-14T14:51:55,014 INFO [RS:2;da1c6afcd1f9:46007 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=da1c6afcd1f9%2C46007%2C1731595913586, suffix=, logDir=hdfs://localhost:34445/user/jenkins/test-data/35161b07-018a-0344-8c43-1519f3812567/WALs/da1c6afcd1f9,46007,1731595913586, archiveDir=hdfs://localhost:34445/user/jenkins/test-data/35161b07-018a-0344-8c43-1519f3812567/oldWALs, maxLogs=32 2024-11-14T14:51:55,033 DEBUG [RS:2;da1c6afcd1f9:46007 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/35161b07-018a-0344-8c43-1519f3812567/WALs/da1c6afcd1f9,46007,1731595913586/da1c6afcd1f9%2C46007%2C1731595913586.1731595915018, exclude list is [], retry=0 2024-11-14T14:51:55,033 DEBUG [RS:0;da1c6afcd1f9:37695 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/35161b07-018a-0344-8c43-1519f3812567/WALs/da1c6afcd1f9,37695,1731595913427/da1c6afcd1f9%2C37695%2C1731595913427.1731595915018, exclude list is [], retry=0 2024-11-14T14:51:55,033 DEBUG [RS:1;da1c6afcd1f9:34155 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/35161b07-018a-0344-8c43-1519f3812567/WALs/da1c6afcd1f9,34155,1731595913533/da1c6afcd1f9%2C34155%2C1731595913533.1731595915018, exclude list is [], retry=0 2024-11-14T14:51:55,039 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36189,DS-5c245456-9209-4937-bfad-43a9a1b09820,DISK] 2024-11-14T14:51:55,039 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36189,DS-5c245456-9209-4937-bfad-43a9a1b09820,DISK] 2024-11-14T14:51:55,039 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42819,DS-c1b8ccdc-5e2c-4b3d-bb1e-6941b40dfb7e,DISK] 2024-11-14T14:51:55,039 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42819,DS-c1b8ccdc-5e2c-4b3d-bb1e-6941b40dfb7e,DISK] 2024-11-14T14:51:55,040 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44549,DS-913adb4a-7946-4f46-896c-56bcdc4fcfa8,DISK] 2024-11-14T14:51:55,040 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44549,DS-913adb4a-7946-4f46-896c-56bcdc4fcfa8,DISK] 2024-11-14T14:51:55,040 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36189,DS-5c245456-9209-4937-bfad-43a9a1b09820,DISK] 2024-11-14T14:51:55,040 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42819,DS-c1b8ccdc-5e2c-4b3d-bb1e-6941b40dfb7e,DISK] 2024-11-14T14:51:55,041 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44549,DS-913adb4a-7946-4f46-896c-56bcdc4fcfa8,DISK] 2024-11-14T14:51:55,084 INFO [RS:0;da1c6afcd1f9:37695 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/35161b07-018a-0344-8c43-1519f3812567/WALs/da1c6afcd1f9,37695,1731595913427/da1c6afcd1f9%2C37695%2C1731595913427.1731595915018 2024-11-14T14:51:55,085 DEBUG [RS:0;da1c6afcd1f9:37695 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33623:33623),(127.0.0.1/127.0.0.1:33515:33515),(127.0.0.1/127.0.0.1:34777:34777)] 2024-11-14T14:51:55,094 INFO [RS:1;da1c6afcd1f9:34155 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/35161b07-018a-0344-8c43-1519f3812567/WALs/da1c6afcd1f9,34155,1731595913533/da1c6afcd1f9%2C34155%2C1731595913533.1731595915018 2024-11-14T14:51:55,094 INFO [RS:2;da1c6afcd1f9:46007 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/35161b07-018a-0344-8c43-1519f3812567/WALs/da1c6afcd1f9,46007,1731595913586/da1c6afcd1f9%2C46007%2C1731595913586.1731595915018 2024-11-14T14:51:55,096 DEBUG [RS:1;da1c6afcd1f9:34155 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34777:34777),(127.0.0.1/127.0.0.1:33623:33623),(127.0.0.1/127.0.0.1:33515:33515)] 2024-11-14T14:51:55,096 DEBUG [RS:2;da1c6afcd1f9:46007 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33623:33623),(127.0.0.1/127.0.0.1:34777:34777),(127.0.0.1/127.0.0.1:33515:33515)] 2024-11-14T14:51:55,200 DEBUG [da1c6afcd1f9:42151 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=3, allServersCount=3 2024-11-14T14:51:55,209 DEBUG [da1c6afcd1f9:42151 {}] balancer.BalancerClusterState(204): Hosts are {da1c6afcd1f9=0} racks are {/default-rack=0} 2024-11-14T14:51:55,216 DEBUG [da1c6afcd1f9:42151 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-14T14:51:55,216 DEBUG [da1c6afcd1f9:42151 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-14T14:51:55,216 DEBUG [da1c6afcd1f9:42151 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-11-14T14:51:55,217 DEBUG [da1c6afcd1f9:42151 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-14T14:51:55,217 DEBUG [da1c6afcd1f9:42151 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-14T14:51:55,217 DEBUG [da1c6afcd1f9:42151 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-11-14T14:51:55,217 INFO [da1c6afcd1f9:42151 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-14T14:51:55,217 INFO [da1c6afcd1f9:42151 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-14T14:51:55,217 INFO [da1c6afcd1f9:42151 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-11-14T14:51:55,217 DEBUG [da1c6afcd1f9:42151 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-14T14:51:55,224 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=da1c6afcd1f9,46007,1731595913586 2024-11-14T14:51:55,230 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as da1c6afcd1f9,46007,1731595913586, state=OPENING 2024-11-14T14:51:55,235 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-14T14:51:55,237 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46007-0x101a74438830003, quorum=127.0.0.1:51231, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T14:51:55,237 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42151-0x101a74438830000, quorum=127.0.0.1:51231, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T14:51:55,237 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37695-0x101a74438830001, quorum=127.0.0.1:51231, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T14:51:55,237 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34155-0x101a74438830002, quorum=127.0.0.1:51231, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T14:51:55,238 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T14:51:55,238 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T14:51:55,238 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T14:51:55,238 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T14:51:55,240 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-14T14:51:55,242 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=da1c6afcd1f9,46007,1731595913586}] 2024-11-14T14:51:55,419 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-14T14:51:55,421 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38693, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-14T14:51:55,433 INFO [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-14T14:51:55,434 INFO [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-14T14:51:55,435 INFO [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-14T14:51:55,438 INFO [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=da1c6afcd1f9%2C46007%2C1731595913586.meta, suffix=.meta, logDir=hdfs://localhost:34445/user/jenkins/test-data/35161b07-018a-0344-8c43-1519f3812567/WALs/da1c6afcd1f9,46007,1731595913586, archiveDir=hdfs://localhost:34445/user/jenkins/test-data/35161b07-018a-0344-8c43-1519f3812567/oldWALs, maxLogs=32 2024-11-14T14:51:55,454 DEBUG [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/35161b07-018a-0344-8c43-1519f3812567/WALs/da1c6afcd1f9,46007,1731595913586/da1c6afcd1f9%2C46007%2C1731595913586.meta.1731595915439.meta, exclude list is [], retry=0 2024-11-14T14:51:55,457 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36189,DS-5c245456-9209-4937-bfad-43a9a1b09820,DISK] 2024-11-14T14:51:55,457 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44549,DS-913adb4a-7946-4f46-896c-56bcdc4fcfa8,DISK] 2024-11-14T14:51:55,457 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42819,DS-c1b8ccdc-5e2c-4b3d-bb1e-6941b40dfb7e,DISK] 2024-11-14T14:51:55,489 INFO [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/35161b07-018a-0344-8c43-1519f3812567/WALs/da1c6afcd1f9,46007,1731595913586/da1c6afcd1f9%2C46007%2C1731595913586.meta.1731595915439.meta 2024-11-14T14:51:55,490 DEBUG [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33623:33623),(127.0.0.1/127.0.0.1:34777:34777),(127.0.0.1/127.0.0.1:33515:33515)] 2024-11-14T14:51:55,490 DEBUG [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-14T14:51:55,492 DEBUG [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-14T14:51:55,495 DEBUG [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-14T14:51:55,500 INFO [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-14T14:51:55,505 DEBUG [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-14T14:51:55,505 DEBUG [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T14:51:55,505 DEBUG [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-14T14:51:55,505 DEBUG [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-14T14:51:55,508 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-14T14:51:55,510 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-14T14:51:55,510 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T14:51:55,511 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T14:51:55,511 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-14T14:51:55,512 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-14T14:51:55,512 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T14:51:55,513 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T14:51:55,513 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-14T14:51:55,514 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-14T14:51:55,514 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T14:51:55,515 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T14:51:55,515 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-14T14:51:55,516 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-14T14:51:55,517 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T14:51:55,517 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T14:51:55,518 DEBUG [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-14T14:51:55,519 DEBUG [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34445/user/jenkins/test-data/35161b07-018a-0344-8c43-1519f3812567/data/hbase/meta/1588230740 2024-11-14T14:51:55,522 DEBUG [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34445/user/jenkins/test-data/35161b07-018a-0344-8c43-1519f3812567/data/hbase/meta/1588230740 2024-11-14T14:51:55,524 DEBUG [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-14T14:51:55,524 DEBUG [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-14T14:51:55,525 DEBUG [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-14T14:51:55,528 DEBUG [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-14T14:51:55,529 INFO [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68121005, jitterRate=0.01508207619190216}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-14T14:51:55,529 DEBUG [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-14T14:51:55,531 DEBUG [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731595915506Writing region info on filesystem at 1731595915506Initializing all the Stores at 1731595915508 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731595915508Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731595915508Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731595915508Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731595915508Cleaning up temporary data from old regions at 1731595915524 (+16 ms)Running coprocessor post-open hooks at 1731595915529 (+5 ms)Region opened successfully at 1731595915531 (+2 ms) 2024-11-14T14:51:55,539 INFO [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731595915410 2024-11-14T14:51:55,551 DEBUG [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-14T14:51:55,552 INFO [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-14T14:51:55,554 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=da1c6afcd1f9,46007,1731595913586 2024-11-14T14:51:55,556 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as da1c6afcd1f9,46007,1731595913586, state=OPEN 2024-11-14T14:51:55,558 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46007-0x101a74438830003, quorum=127.0.0.1:51231, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-14T14:51:55,558 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37695-0x101a74438830001, quorum=127.0.0.1:51231, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-14T14:51:55,558 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34155-0x101a74438830002, quorum=127.0.0.1:51231, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-14T14:51:55,559 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T14:51:55,559 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42151-0x101a74438830000, quorum=127.0.0.1:51231, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-14T14:51:55,559 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T14:51:55,559 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T14:51:55,559 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T14:51:55,559 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=da1c6afcd1f9,46007,1731595913586 2024-11-14T14:51:55,566 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-14T14:51:55,566 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=da1c6afcd1f9,46007,1731595913586 in 318 msec 2024-11-14T14:51:55,573 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-14T14:51:55,574 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 786 msec 2024-11-14T14:51:55,575 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-14T14:51:55,575 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-14T14:51:55,620 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T14:51:55,622 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=da1c6afcd1f9,46007,1731595913586, seqNum=-1] 2024-11-14T14:51:55,651 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T14:51:55,654 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41525, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T14:51:55,676 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.1150 sec 2024-11-14T14:51:55,676 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731595915676, completionTime=-1 2024-11-14T14:51:55,678 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=3; waited=0ms, expected min=3 server(s), max=3 server(s), master is running 2024-11-14T14:51:55,678 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-14T14:51:55,705 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=3 2024-11-14T14:51:55,705 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731595975705 2024-11-14T14:51:55,705 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731596035705 2024-11-14T14:51:55,705 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 26 msec 2024-11-14T14:51:55,707 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-11-14T14:51:55,713 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=da1c6afcd1f9,42151,1731595912734-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-14T14:51:55,714 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=da1c6afcd1f9,42151,1731595912734-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-14T14:51:55,714 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=da1c6afcd1f9,42151,1731595912734-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-14T14:51:55,715 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-da1c6afcd1f9:42151, period=300000, unit=MILLISECONDS is enabled. 2024-11-14T14:51:55,715 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-14T14:51:55,716 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-14T14:51:55,722 DEBUG [master/da1c6afcd1f9:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-14T14:51:55,752 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 2.082sec 2024-11-14T14:51:55,753 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-14T14:51:55,755 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-14T14:51:55,755 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-14T14:51:55,756 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-14T14:51:55,756 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-14T14:51:55,757 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=da1c6afcd1f9,42151,1731595912734-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-14T14:51:55,757 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=da1c6afcd1f9,42151,1731595912734-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-14T14:51:55,762 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-14T14:51:55,762 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-14T14:51:55,763 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=da1c6afcd1f9,42151,1731595912734-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-14T14:51:55,834 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4810dfd4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T14:51:55,835 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request da1c6afcd1f9,42151,-1 for getting cluster id 2024-11-14T14:51:55,839 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T14:51:55,850 DEBUG [HMaster-EventLoopGroup-2-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '72d32361-5e69-4ee6-9ad5-db998b6b568a' 2024-11-14T14:51:55,852 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T14:51:55,852 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "72d32361-5e69-4ee6-9ad5-db998b6b568a" 2024-11-14T14:51:55,853 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@311e8e65, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T14:51:55,853 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [da1c6afcd1f9,42151,-1] 2024-11-14T14:51:55,856 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T14:51:55,857 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T14:51:55,858 INFO [HMaster-EventLoopGroup-2-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41770, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T14:51:55,861 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@91db148, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T14:51:55,861 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T14:51:55,868 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=da1c6afcd1f9,46007,1731595913586, seqNum=-1] 2024-11-14T14:51:55,868 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T14:51:55,870 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53148, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T14:51:55,897 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=da1c6afcd1f9,42151,1731595912734 2024-11-14T14:51:55,898 INFO [Time-limited test {}] wal.AbstractTestWALReplay(147): hbase.rootdir=hdfs://localhost:34445/hbase 2024-11-14T14:51:55,918 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testNameConflictWhenSplit0 Thread=365, OpenFileDescriptor=607, MaxFileDescriptor=1048576, SystemLoadAverage=199, ProcessCount=11, AvailableMemoryMB=7989 2024-11-14T14:51:55,937 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-14T14:51:55,940 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-14T14:51:55,940 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-14T14:51:55,945 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-44990049, suffix=, logDir=hdfs://localhost:34445/hbase/WALs/hregion-44990049, archiveDir=hdfs://localhost:34445/hbase/oldWALs, maxLogs=32 2024-11-14T14:51:55,961 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-44990049/hregion-44990049.1731595915947, exclude list is [], retry=0 2024-11-14T14:51:55,965 DEBUG [AsyncFSWAL-8-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44549,DS-913adb4a-7946-4f46-896c-56bcdc4fcfa8,DISK] 2024-11-14T14:51:55,966 DEBUG [AsyncFSWAL-8-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36189,DS-5c245456-9209-4937-bfad-43a9a1b09820,DISK] 2024-11-14T14:51:55,966 DEBUG [AsyncFSWAL-8-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42819,DS-c1b8ccdc-5e2c-4b3d-bb1e-6941b40dfb7e,DISK] 2024-11-14T14:51:55,996 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-44990049/hregion-44990049.1731595915947 2024-11-14T14:51:55,997 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33515:33515),(127.0.0.1/127.0.0.1:33623:33623),(127.0.0.1/127.0.0.1:34777:34777)] 2024-11-14T14:51:55,997 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 5b51d66043cb429bbbe77aa23201a420, NAME => 'testReplayEditsWrittenIntoWAL,,1731595915937.5b51d66043cb429bbbe77aa23201a420.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenIntoWAL', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34445/hbase 2024-11-14T14:51:56,010 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36189 is added to blk_1073741838_1014 (size=64) 2024-11-14T14:51:56,010 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44549 is added to blk_1073741838_1014 (size=64) 2024-11-14T14:51:56,010 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42819 is added to blk_1073741838_1014 (size=64) 2024-11-14T14:51:56,011 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1731595915937.5b51d66043cb429bbbe77aa23201a420.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T14:51:56,013 INFO [StoreOpener-5b51d66043cb429bbbe77aa23201a420-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 5b51d66043cb429bbbe77aa23201a420 2024-11-14T14:51:56,016 INFO [StoreOpener-5b51d66043cb429bbbe77aa23201a420-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5b51d66043cb429bbbe77aa23201a420 columnFamilyName a 2024-11-14T14:51:56,016 DEBUG [StoreOpener-5b51d66043cb429bbbe77aa23201a420-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T14:51:56,017 INFO [StoreOpener-5b51d66043cb429bbbe77aa23201a420-1 {}] regionserver.HStore(327): Store=5b51d66043cb429bbbe77aa23201a420/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T14:51:56,017 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 5b51d66043cb429bbbe77aa23201a420 2024-11-14T14:51:56,018 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34445/hbase/data/default/testReplayEditsWrittenIntoWAL/5b51d66043cb429bbbe77aa23201a420 2024-11-14T14:51:56,019 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34445/hbase/data/default/testReplayEditsWrittenIntoWAL/5b51d66043cb429bbbe77aa23201a420 2024-11-14T14:51:56,020 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 5b51d66043cb429bbbe77aa23201a420 2024-11-14T14:51:56,020 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 5b51d66043cb429bbbe77aa23201a420 2024-11-14T14:51:56,022 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 5b51d66043cb429bbbe77aa23201a420 2024-11-14T14:51:56,026 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34445/hbase/data/default/testReplayEditsWrittenIntoWAL/5b51d66043cb429bbbe77aa23201a420/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-14T14:51:56,027 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 5b51d66043cb429bbbe77aa23201a420; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63819961, jitterRate=-0.04900847375392914}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-14T14:51:56,029 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 5b51d66043cb429bbbe77aa23201a420: Writing region info on filesystem at 1731595916011Initializing all the Stores at 1731595916013 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731595916013Cleaning up temporary data from old regions at 1731595916020 (+7 ms)Region opened successfully at 1731595916028 (+8 ms) 2024-11-14T14:51:56,029 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 5b51d66043cb429bbbe77aa23201a420, disabling compactions & flushes 2024-11-14T14:51:56,029 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1731595915937.5b51d66043cb429bbbe77aa23201a420. 2024-11-14T14:51:56,029 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1731595915937.5b51d66043cb429bbbe77aa23201a420. 2024-11-14T14:51:56,029 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1731595915937.5b51d66043cb429bbbe77aa23201a420. after waiting 0 ms 2024-11-14T14:51:56,029 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1731595915937.5b51d66043cb429bbbe77aa23201a420. 2024-11-14T14:51:56,029 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1731595915937.5b51d66043cb429bbbe77aa23201a420. 2024-11-14T14:51:56,029 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 5b51d66043cb429bbbe77aa23201a420: Waiting for close lock at 1731595916029Disabling compacts and flushes for region at 1731595916029Disabling writes for close at 1731595916029Writing region close event to WAL at 1731595916029Closed at 1731595916029 2024-11-14T14:51:56,035 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42819 is added to blk_1073741837_1013 (size=175) 2024-11-14T14:51:56,035 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44549 is added to blk_1073741837_1013 (size=175) 2024-11-14T14:51:56,035 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36189 is added to blk_1073741837_1013 (size=175) 2024-11-14T14:51:56,042 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-14T14:51:56,043 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-44990049:(num 1731595915947) 2024-11-14T14:51:56,044 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-11-14T14:51:56,091 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44549 is added to blk_1073741839_1015 (size=469) 2024-11-14T14:51:56,091 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36189 is added to blk_1073741839_1015 (size=469) 2024-11-14T14:51:56,091 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42819 is added to blk_1073741839_1015 (size=469) 2024-11-14T14:51:56,093 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-11-14T14:51:56,133 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44549 is added to blk_1073741840_1016 (size=322) 2024-11-14T14:51:56,133 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36189 is added to blk_1073741840_1016 (size=322) 2024-11-14T14:51:56,134 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42819 is added to blk_1073741840_1016 (size=322) 2024-11-14T14:51:56,158 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:34445/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731595915935/wal-1, size=469 (469bytes) 2024-11-14T14:51:56,159 DEBUG [Time-limited test {}] util.RecoverLeaseFSUtils(47): Initialize RecoverLeaseFSUtils 2024-11-14T14:51:56,159 DEBUG [Time-limited test {}] util.RecoverLeaseFSUtils(59): set recoverLeaseMethod to org.apache.hadoop.fs.LeaseRecoverable.recoverLease() 2024-11-14T14:51:56,159 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:34445/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731595915935/wal-1 2024-11-14T14:51:56,166 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:34445/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731595915935/wal-1 after 4ms 2024-11-14T14:51:56,199 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-14T14:51:56,200 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:34445/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731595915935/wal-1 took 44ms 2024-11-14T14:51:56,207 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:34445/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731595915935/wal-1 so closing down 2024-11-14T14:51:56,207 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-14T14:51:56,210 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal-1.temp 2024-11-14T14:51:56,239 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:34445/hbase/data/default/testReplayEditsWrittenIntoWAL/5b51d66043cb429bbbe77aa23201a420/recovered.edits/0000000000000000001-wal-1.temp 2024-11-14T14:51:56,240 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-14T14:51:56,250 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36189 is added to blk_1073741841_1017 (size=469) 2024-11-14T14:51:56,250 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42819 is added to blk_1073741841_1017 (size=469) 2024-11-14T14:51:56,251 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44549 is added to blk_1073741841_1017 (size=469) 2024-11-14T14:51:56,254 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:34445/hbase/data/default/testReplayEditsWrittenIntoWAL/5b51d66043cb429bbbe77aa23201a420/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) 2024-11-14T14:51:56,256 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:34445/hbase/data/default/testReplayEditsWrittenIntoWAL/5b51d66043cb429bbbe77aa23201a420/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:34445/hbase/data/default/testReplayEditsWrittenIntoWAL/5b51d66043cb429bbbe77aa23201a420/recovered.edits/0000000000000000002 2024-11-14T14:51:56,262 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 2 edits across 1 Regions in 55 ms; skipped=0; WAL=hdfs://localhost:34445/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731595915935/wal-1, size=469, length=469, corrupted=false, cancelled=false 2024-11-14T14:51:56,262 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:34445/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731595915935/wal-1, journal: Splitting hdfs://localhost:34445/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731595915935/wal-1, size=469 (469bytes) at 1731595916158Finishing writing output for hdfs://localhost:34445/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731595915935/wal-1 so closing down at 1731595916207 (+49 ms)Creating recovered edits writer path=hdfs://localhost:34445/hbase/data/default/testReplayEditsWrittenIntoWAL/5b51d66043cb429bbbe77aa23201a420/recovered.edits/0000000000000000001-wal-1.temp at 1731595916239 (+32 ms)3 split writer threads finished at 1731595916241 (+2 ms)Closed recovered edits writer path=hdfs://localhost:34445/hbase/data/default/testReplayEditsWrittenIntoWAL/5b51d66043cb429bbbe77aa23201a420/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) at 1731595916254 (+13 ms)Rename recovered edits hdfs://localhost:34445/hbase/data/default/testReplayEditsWrittenIntoWAL/5b51d66043cb429bbbe77aa23201a420/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:34445/hbase/data/default/testReplayEditsWrittenIntoWAL/5b51d66043cb429bbbe77aa23201a420/recovered.edits/0000000000000000002 at 1731595916257 (+3 ms)Processed 2 edits across 1 Regions in 55 ms; skipped=0; WAL=hdfs://localhost:34445/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731595915935/wal-1, size=469, length=469, corrupted=false, cancelled=false at 1731595916262 (+5 ms) 2024-11-14T14:51:56,286 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:34445/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731595915935/wal-2, size=322 (322bytes) 2024-11-14T14:51:56,287 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:34445/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731595915935/wal-2 2024-11-14T14:51:56,288 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:34445/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731595915935/wal-2 after 0ms 2024-11-14T14:51:56,322 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-14T14:51:56,322 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:34445/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731595915935/wal-2 took 36ms 2024-11-14T14:51:56,325 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:34445/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731595915935/wal-2 so closing down 2024-11-14T14:51:56,325 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-14T14:51:56,328 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000002-wal-2.temp 2024-11-14T14:51:56,356 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:34445/hbase/data/default/testReplayEditsWrittenIntoWAL/5b51d66043cb429bbbe77aa23201a420/recovered.edits/0000000000000000002-wal-2.temp 2024-11-14T14:51:56,357 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-14T14:51:56,366 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44549 is added to blk_1073741842_1018 (size=322) 2024-11-14T14:51:56,366 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42819 is added to blk_1073741842_1018 (size=322) 2024-11-14T14:51:56,366 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36189 is added to blk_1073741842_1018 (size=322) 2024-11-14T14:51:56,368 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:34445/hbase/data/default/testReplayEditsWrittenIntoWAL/5b51d66043cb429bbbe77aa23201a420/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-11-14T14:51:56,398 DEBUG [split-log-closeStream-pool-0 {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-14T14:51:56,401 WARN [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(184): Found existing old edits file and we have less entries. Deleting hdfs://localhost:34445/hbase/data/default/testReplayEditsWrittenIntoWAL/5b51d66043cb429bbbe77aa23201a420/recovered.edits/0000000000000000002-wal-2.temp, length=322 2024-11-14T14:51:56,402 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 1 edits across 1 Regions in 80 ms; skipped=0; WAL=hdfs://localhost:34445/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731595915935/wal-2, size=322, length=322, corrupted=false, cancelled=false 2024-11-14T14:51:56,403 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:34445/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731595915935/wal-2, journal: Splitting hdfs://localhost:34445/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731595915935/wal-2, size=322 (322bytes) at 1731595916286Finishing writing output for hdfs://localhost:34445/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731595915935/wal-2 so closing down at 1731595916325 (+39 ms)Creating recovered edits writer path=hdfs://localhost:34445/hbase/data/default/testReplayEditsWrittenIntoWAL/5b51d66043cb429bbbe77aa23201a420/recovered.edits/0000000000000000002-wal-2.temp at 1731595916356 (+31 ms)3 split writer threads finished at 1731595916357 (+1 ms)Closed recovered edits writer path=hdfs://localhost:34445/hbase/data/default/testReplayEditsWrittenIntoWAL/5b51d66043cb429bbbe77aa23201a420/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1731595916368 (+11 ms)Processed 1 edits across 1 Regions in 80 ms; skipped=0; WAL=hdfs://localhost:34445/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731595915935/wal-2, size=322, length=322, corrupted=false, cancelled=false at 1731595916403 (+35 ms) 2024-11-14T14:51:56,403 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-14T14:51:56,406 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:34445/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731595915935, archiveDir=hdfs://localhost:34445/hbase/oldWALs, maxLogs=32 2024-11-14T14:51:56,420 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testnameconflictwhensplit0-manual,16010,1731595915935/wal.1731595916407, exclude list is [], retry=0 2024-11-14T14:51:56,424 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36189,DS-5c245456-9209-4937-bfad-43a9a1b09820,DISK] 2024-11-14T14:51:56,425 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44549,DS-913adb4a-7946-4f46-896c-56bcdc4fcfa8,DISK] 2024-11-14T14:51:56,425 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42819,DS-c1b8ccdc-5e2c-4b3d-bb1e-6941b40dfb7e,DISK] 2024-11-14T14:51:56,454 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testnameconflictwhensplit0-manual,16010,1731595915935/wal.1731595916407 2024-11-14T14:51:56,454 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33623:33623),(127.0.0.1/127.0.0.1:33515:33515),(127.0.0.1/127.0.0.1:34777:34777)] 2024-11-14T14:51:56,454 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 5b51d66043cb429bbbe77aa23201a420, NAME => 'testReplayEditsWrittenIntoWAL,,1731595915937.5b51d66043cb429bbbe77aa23201a420.', STARTKEY => '', ENDKEY => ''} 2024-11-14T14:51:56,455 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1731595915937.5b51d66043cb429bbbe77aa23201a420.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T14:51:56,455 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 5b51d66043cb429bbbe77aa23201a420 2024-11-14T14:51:56,455 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 5b51d66043cb429bbbe77aa23201a420 2024-11-14T14:51:56,457 INFO [StoreOpener-5b51d66043cb429bbbe77aa23201a420-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 5b51d66043cb429bbbe77aa23201a420 2024-11-14T14:51:56,459 INFO [StoreOpener-5b51d66043cb429bbbe77aa23201a420-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5b51d66043cb429bbbe77aa23201a420 columnFamilyName a 2024-11-14T14:51:56,459 DEBUG [StoreOpener-5b51d66043cb429bbbe77aa23201a420-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T14:51:56,460 INFO [StoreOpener-5b51d66043cb429bbbe77aa23201a420-1 {}] regionserver.HStore(327): Store=5b51d66043cb429bbbe77aa23201a420/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T14:51:56,460 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 5b51d66043cb429bbbe77aa23201a420 2024-11-14T14:51:56,461 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34445/hbase/data/default/testReplayEditsWrittenIntoWAL/5b51d66043cb429bbbe77aa23201a420 2024-11-14T14:51:56,464 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:34445/hbase/data/default/testReplayEditsWrittenIntoWAL/5b51d66043cb429bbbe77aa23201a420 2024-11-14T14:51:56,465 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:34445/hbase/data/default/testReplayEditsWrittenIntoWAL/5b51d66043cb429bbbe77aa23201a420/recovered.edits/0000000000000000002 2024-11-14T14:51:56,498 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-14T14:51:56,505 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 2, skipped 0, firstSequenceIdInLog=1, maxSequenceIdInLog=2, path=hdfs://localhost:34445/hbase/data/default/testReplayEditsWrittenIntoWAL/5b51d66043cb429bbbe77aa23201a420/recovered.edits/0000000000000000002 2024-11-14T14:51:56,509 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 5b51d66043cb429bbbe77aa23201a420 1/1 column families, dataSize=108 B heapSize=512 B 2024-11-14T14:51:56,572 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/hbase/data/default/testReplayEditsWrittenIntoWAL/5b51d66043cb429bbbe77aa23201a420/.tmp/a/68f6b49693fb4d76a545c9b333afa149 is 58, key is testReplayEditsWrittenIntoWAL/a:1/1731595916043/Put/seqid=0 2024-11-14T14:51:56,587 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44549 is added to blk_1073741844_1020 (size=5170) 2024-11-14T14:51:56,587 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42819 is added to blk_1073741844_1020 (size=5170) 2024-11-14T14:51:56,587 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36189 is added to blk_1073741844_1020 (size=5170) 2024-11-14T14:51:56,588 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=108 B at sequenceid=2 (bloomFilter=true), to=hdfs://localhost:34445/hbase/data/default/testReplayEditsWrittenIntoWAL/5b51d66043cb429bbbe77aa23201a420/.tmp/a/68f6b49693fb4d76a545c9b333afa149 2024-11-14T14:51:56,635 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/hbase/data/default/testReplayEditsWrittenIntoWAL/5b51d66043cb429bbbe77aa23201a420/.tmp/a/68f6b49693fb4d76a545c9b333afa149 as hdfs://localhost:34445/hbase/data/default/testReplayEditsWrittenIntoWAL/5b51d66043cb429bbbe77aa23201a420/a/68f6b49693fb4d76a545c9b333afa149 2024-11-14T14:51:56,646 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/hbase/data/default/testReplayEditsWrittenIntoWAL/5b51d66043cb429bbbe77aa23201a420/a/68f6b49693fb4d76a545c9b333afa149, entries=2, sequenceid=2, filesize=5.0 K 2024-11-14T14:51:56,653 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for 5b51d66043cb429bbbe77aa23201a420 in 142ms, sequenceid=2, compaction requested=false; wal=null 2024-11-14T14:51:56,655 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:34445/hbase/data/default/testReplayEditsWrittenIntoWAL/5b51d66043cb429bbbe77aa23201a420/recovered.edits/0000000000000000002 2024-11-14T14:51:56,656 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 5b51d66043cb429bbbe77aa23201a420 2024-11-14T14:51:56,656 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 5b51d66043cb429bbbe77aa23201a420 2024-11-14T14:51:56,660 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 5b51d66043cb429bbbe77aa23201a420 2024-11-14T14:51:56,663 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34445/hbase/data/default/testReplayEditsWrittenIntoWAL/5b51d66043cb429bbbe77aa23201a420/recovered.edits/2.seqid, newMaxSeqId=2, maxSeqId=1 2024-11-14T14:51:56,665 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 5b51d66043cb429bbbe77aa23201a420; next sequenceid=3; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67378770, jitterRate=0.0040219128131866455}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-14T14:51:56,666 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 5b51d66043cb429bbbe77aa23201a420: Writing region info on filesystem at 1731595916455Initializing all the Stores at 1731595916457 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731595916457Obtaining lock to block concurrent updates at 1731595916509 (+52 ms)Preparing flush snapshotting stores in 5b51d66043cb429bbbe77aa23201a420 at 1731595916509Finished memstore snapshotting testReplayEditsWrittenIntoWAL,,1731595915937.5b51d66043cb429bbbe77aa23201a420., syncing WAL and waiting on mvcc, flushsize=dataSize=108, getHeapSize=496, getOffHeapSize=0, getCellsCount=2 at 1731595916514 (+5 ms)Flushing stores of testReplayEditsWrittenIntoWAL,,1731595915937.5b51d66043cb429bbbe77aa23201a420. at 1731595916514Flushing 5b51d66043cb429bbbe77aa23201a420/a: creating writer at 1731595916515 (+1 ms)Flushing 5b51d66043cb429bbbe77aa23201a420/a: appending metadata at 1731595916561 (+46 ms)Flushing 5b51d66043cb429bbbe77aa23201a420/a: closing flushed file at 1731595916563 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6a42d9b4: reopening flushed file at 1731595916633 (+70 ms)Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for 5b51d66043cb429bbbe77aa23201a420 in 142ms, sequenceid=2, compaction requested=false; wal=null at 1731595916654 (+21 ms)Cleaning up temporary data from old regions at 1731595916656 (+2 ms)Region opened successfully at 1731595916666 (+10 ms) 2024-11-14T14:51:56,693 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testNameConflictWhenSplit0 Thread=377 (was 365) Potentially hanging thread: PacketResponder: BP-1055376130-172.17.0.2-1731595909704:blk_1073741843_1019, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1791616411_22 at /127.0.0.1:55398 [Waiting for operation #8] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1791616411_22 at /127.0.0.1:36692 [Waiting for operation #10] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1055376130-172.17.0.2-1731595909704:blk_1073741843_1019, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1791616411_22 at /127.0.0.1:36738 [Receiving block BP-1055376130-172.17.0.2-1731595909704:blk_1073741843_1019] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-8-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-8-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-0-hdfs://localhost:34445/hbase-prefix:default java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1055376130-172.17.0.2-1731595909704:blk_1073741843_1019, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-8-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1791616411_22 at /127.0.0.1:34314 [Receiving block BP-1055376130-172.17.0.2-1731595909704:blk_1073741843_1019] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: TestAsyncWALReplay-pool-0 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1791616411_22 at /127.0.0.1:55396 [Receiving block BP-1055376130-172.17.0.2-1731595909704:blk_1073741843_1019] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=703 (was 607) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=199 (was 199), ProcessCount=11 (was 11), AvailableMemoryMB=7977 (was 7989) 2024-11-14T14:51:56,707 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testNameConflictWhenSplit1 Thread=377, OpenFileDescriptor=703, MaxFileDescriptor=1048576, SystemLoadAverage=199, ProcessCount=11, AvailableMemoryMB=7973 2024-11-14T14:51:56,723 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-14T14:51:56,725 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-14T14:51:56,726 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-14T14:51:56,729 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-20352620, suffix=, logDir=hdfs://localhost:34445/hbase/WALs/hregion-20352620, archiveDir=hdfs://localhost:34445/hbase/oldWALs, maxLogs=32 2024-11-14T14:51:56,743 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-20352620/hregion-20352620.1731595916730, exclude list is [], retry=0 2024-11-14T14:51:56,748 DEBUG [AsyncFSWAL-10-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42819,DS-c1b8ccdc-5e2c-4b3d-bb1e-6941b40dfb7e,DISK] 2024-11-14T14:51:56,748 DEBUG [AsyncFSWAL-10-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44549,DS-913adb4a-7946-4f46-896c-56bcdc4fcfa8,DISK] 2024-11-14T14:51:56,749 DEBUG [AsyncFSWAL-10-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36189,DS-5c245456-9209-4937-bfad-43a9a1b09820,DISK] 2024-11-14T14:51:56,779 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-20352620/hregion-20352620.1731595916730 2024-11-14T14:51:56,780 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34777:34777),(127.0.0.1/127.0.0.1:33515:33515),(127.0.0.1/127.0.0.1:33623:33623)] 2024-11-14T14:51:56,780 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 971f94bfe32ca05e7c77cc84c8a61347, NAME => 'testReplayEditsWrittenIntoWAL,,1731595916723.971f94bfe32ca05e7c77cc84c8a61347.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenIntoWAL', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34445/hbase 2024-11-14T14:51:56,792 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42819 is added to blk_1073741846_1022 (size=64) 2024-11-14T14:51:56,793 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36189 is added to blk_1073741846_1022 (size=64) 2024-11-14T14:51:56,793 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44549 is added to blk_1073741846_1022 (size=64) 2024-11-14T14:51:56,794 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1731595916723.971f94bfe32ca05e7c77cc84c8a61347.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T14:51:56,796 INFO [StoreOpener-971f94bfe32ca05e7c77cc84c8a61347-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 971f94bfe32ca05e7c77cc84c8a61347 2024-11-14T14:51:56,799 INFO [StoreOpener-971f94bfe32ca05e7c77cc84c8a61347-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 971f94bfe32ca05e7c77cc84c8a61347 columnFamilyName a 2024-11-14T14:51:56,799 DEBUG [StoreOpener-971f94bfe32ca05e7c77cc84c8a61347-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T14:51:56,800 INFO [StoreOpener-971f94bfe32ca05e7c77cc84c8a61347-1 {}] regionserver.HStore(327): Store=971f94bfe32ca05e7c77cc84c8a61347/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T14:51:56,800 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 971f94bfe32ca05e7c77cc84c8a61347 2024-11-14T14:51:56,801 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34445/hbase/data/default/testReplayEditsWrittenIntoWAL/971f94bfe32ca05e7c77cc84c8a61347 2024-11-14T14:51:56,802 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34445/hbase/data/default/testReplayEditsWrittenIntoWAL/971f94bfe32ca05e7c77cc84c8a61347 2024-11-14T14:51:56,802 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 971f94bfe32ca05e7c77cc84c8a61347 2024-11-14T14:51:56,802 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 971f94bfe32ca05e7c77cc84c8a61347 2024-11-14T14:51:56,805 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 971f94bfe32ca05e7c77cc84c8a61347 2024-11-14T14:51:56,808 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34445/hbase/data/default/testReplayEditsWrittenIntoWAL/971f94bfe32ca05e7c77cc84c8a61347/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-14T14:51:56,809 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 971f94bfe32ca05e7c77cc84c8a61347; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63893467, jitterRate=-0.04791314899921417}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-14T14:51:56,809 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 971f94bfe32ca05e7c77cc84c8a61347: Writing region info on filesystem at 1731595916794Initializing all the Stores at 1731595916796 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731595916796Cleaning up temporary data from old regions at 1731595916802 (+6 ms)Region opened successfully at 1731595916809 (+7 ms) 2024-11-14T14:51:56,809 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 971f94bfe32ca05e7c77cc84c8a61347, disabling compactions & flushes 2024-11-14T14:51:56,809 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1731595916723.971f94bfe32ca05e7c77cc84c8a61347. 2024-11-14T14:51:56,809 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1731595916723.971f94bfe32ca05e7c77cc84c8a61347. 2024-11-14T14:51:56,809 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1731595916723.971f94bfe32ca05e7c77cc84c8a61347. after waiting 0 ms 2024-11-14T14:51:56,809 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1731595916723.971f94bfe32ca05e7c77cc84c8a61347. 2024-11-14T14:51:56,809 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1731595916723.971f94bfe32ca05e7c77cc84c8a61347. 2024-11-14T14:51:56,809 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 971f94bfe32ca05e7c77cc84c8a61347: Waiting for close lock at 1731595916809Disabling compacts and flushes for region at 1731595916809Disabling writes for close at 1731595916809Writing region close event to WAL at 1731595916809Closed at 1731595916809 2024-11-14T14:51:56,813 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36189 is added to blk_1073741845_1021 (size=175) 2024-11-14T14:51:56,813 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42819 is added to blk_1073741845_1021 (size=175) 2024-11-14T14:51:56,813 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44549 is added to blk_1073741845_1021 (size=175) 2024-11-14T14:51:56,816 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-14T14:51:56,816 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-20352620:(num 1731595916730) 2024-11-14T14:51:56,817 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-11-14T14:51:56,854 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44549 is added to blk_1073741847_1023 (size=469) 2024-11-14T14:51:56,854 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36189 is added to blk_1073741847_1023 (size=469) 2024-11-14T14:51:56,855 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42819 is added to blk_1073741847_1023 (size=469) 2024-11-14T14:51:56,857 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-11-14T14:51:56,908 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36189 is added to blk_1073741848_1024 (size=322) 2024-11-14T14:51:56,909 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44549 is added to blk_1073741848_1024 (size=322) 2024-11-14T14:51:56,909 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42819 is added to blk_1073741848_1024 (size=322) 2024-11-14T14:51:56,925 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:34445/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731595916722/wal-2, size=322 (322bytes) 2024-11-14T14:51:56,925 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:34445/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731595916722/wal-2 2024-11-14T14:51:56,926 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:34445/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731595916722/wal-2 after 1ms 2024-11-14T14:51:56,955 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-14T14:51:56,955 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:34445/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731595916722/wal-2 took 30ms 2024-11-14T14:51:56,957 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:34445/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731595916722/wal-2 so closing down 2024-11-14T14:51:56,957 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-14T14:51:56,960 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000002-wal-2.temp 2024-11-14T14:51:56,986 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:34445/hbase/data/default/testReplayEditsWrittenIntoWAL/971f94bfe32ca05e7c77cc84c8a61347/recovered.edits/0000000000000000002-wal-2.temp 2024-11-14T14:51:56,987 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-14T14:51:56,995 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36189 is added to blk_1073741849_1025 (size=322) 2024-11-14T14:51:56,995 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42819 is added to blk_1073741849_1025 (size=322) 2024-11-14T14:51:56,996 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44549 is added to blk_1073741849_1025 (size=322) 2024-11-14T14:51:56,996 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:34445/hbase/data/default/testReplayEditsWrittenIntoWAL/971f94bfe32ca05e7c77cc84c8a61347/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-11-14T14:51:56,998 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:34445/hbase/data/default/testReplayEditsWrittenIntoWAL/971f94bfe32ca05e7c77cc84c8a61347/recovered.edits/0000000000000000002-wal-2.temp to hdfs://localhost:34445/hbase/data/default/testReplayEditsWrittenIntoWAL/971f94bfe32ca05e7c77cc84c8a61347/recovered.edits/0000000000000000002 2024-11-14T14:51:56,998 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 1 edits across 1 Regions in 43 ms; skipped=0; WAL=hdfs://localhost:34445/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731595916722/wal-2, size=322, length=322, corrupted=false, cancelled=false 2024-11-14T14:51:56,999 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:34445/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731595916722/wal-2, journal: Splitting hdfs://localhost:34445/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731595916722/wal-2, size=322 (322bytes) at 1731595916925Finishing writing output for hdfs://localhost:34445/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731595916722/wal-2 so closing down at 1731595916957 (+32 ms)Creating recovered edits writer path=hdfs://localhost:34445/hbase/data/default/testReplayEditsWrittenIntoWAL/971f94bfe32ca05e7c77cc84c8a61347/recovered.edits/0000000000000000002-wal-2.temp at 1731595916986 (+29 ms)3 split writer threads finished at 1731595916987 (+1 ms)Closed recovered edits writer path=hdfs://localhost:34445/hbase/data/default/testReplayEditsWrittenIntoWAL/971f94bfe32ca05e7c77cc84c8a61347/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1731595916996 (+9 ms)Rename recovered edits hdfs://localhost:34445/hbase/data/default/testReplayEditsWrittenIntoWAL/971f94bfe32ca05e7c77cc84c8a61347/recovered.edits/0000000000000000002-wal-2.temp to hdfs://localhost:34445/hbase/data/default/testReplayEditsWrittenIntoWAL/971f94bfe32ca05e7c77cc84c8a61347/recovered.edits/0000000000000000002 at 1731595916998 (+2 ms)Processed 1 edits across 1 Regions in 43 ms; skipped=0; WAL=hdfs://localhost:34445/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731595916722/wal-2, size=322, length=322, corrupted=false, cancelled=false at 1731595916998 2024-11-14T14:51:57,012 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:34445/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731595916722/wal-1, size=469 (469bytes) 2024-11-14T14:51:57,012 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:34445/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731595916722/wal-1 2024-11-14T14:51:57,013 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:34445/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731595916722/wal-1 after 1ms 2024-11-14T14:51:57,040 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-14T14:51:57,040 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:34445/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731595916722/wal-1 took 28ms 2024-11-14T14:51:57,043 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:34445/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731595916722/wal-1 so closing down 2024-11-14T14:51:57,043 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-14T14:51:57,045 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal-1.temp 2024-11-14T14:51:57,079 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:34445/hbase/data/default/testReplayEditsWrittenIntoWAL/971f94bfe32ca05e7c77cc84c8a61347/recovered.edits/0000000000000000001-wal-1.temp 2024-11-14T14:51:57,080 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-14T14:51:57,089 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36189 is added to blk_1073741850_1026 (size=469) 2024-11-14T14:51:57,089 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44549 is added to blk_1073741850_1026 (size=469) 2024-11-14T14:51:57,090 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42819 is added to blk_1073741850_1026 (size=469) 2024-11-14T14:51:57,090 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:34445/hbase/data/default/testReplayEditsWrittenIntoWAL/971f94bfe32ca05e7c77cc84c8a61347/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) 2024-11-14T14:51:57,121 DEBUG [split-log-closeStream-pool-0 {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-14T14:51:57,124 WARN [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(175): Found existing old edits file. It could be the result of a previous failed split attempt or we have duplicated wal entries. Deleting hdfs://localhost:34445/hbase/data/default/testReplayEditsWrittenIntoWAL/971f94bfe32ca05e7c77cc84c8a61347/recovered.edits/0000000000000000002, length=322 2024-11-14T14:51:57,128 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:34445/hbase/data/default/testReplayEditsWrittenIntoWAL/971f94bfe32ca05e7c77cc84c8a61347/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:34445/hbase/data/default/testReplayEditsWrittenIntoWAL/971f94bfe32ca05e7c77cc84c8a61347/recovered.edits/0000000000000000002 2024-11-14T14:51:57,128 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 2 edits across 1 Regions in 87 ms; skipped=0; WAL=hdfs://localhost:34445/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731595916722/wal-1, size=469, length=469, corrupted=false, cancelled=false 2024-11-14T14:51:57,128 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:34445/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731595916722/wal-1, journal: Splitting hdfs://localhost:34445/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731595916722/wal-1, size=469 (469bytes) at 1731595917012Finishing writing output for hdfs://localhost:34445/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731595916722/wal-1 so closing down at 1731595917043 (+31 ms)Creating recovered edits writer path=hdfs://localhost:34445/hbase/data/default/testReplayEditsWrittenIntoWAL/971f94bfe32ca05e7c77cc84c8a61347/recovered.edits/0000000000000000001-wal-1.temp at 1731595917079 (+36 ms)3 split writer threads finished at 1731595917080 (+1 ms)Closed recovered edits writer path=hdfs://localhost:34445/hbase/data/default/testReplayEditsWrittenIntoWAL/971f94bfe32ca05e7c77cc84c8a61347/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) at 1731595917090 (+10 ms)Rename recovered edits hdfs://localhost:34445/hbase/data/default/testReplayEditsWrittenIntoWAL/971f94bfe32ca05e7c77cc84c8a61347/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:34445/hbase/data/default/testReplayEditsWrittenIntoWAL/971f94bfe32ca05e7c77cc84c8a61347/recovered.edits/0000000000000000002 at 1731595917128 (+38 ms)Processed 2 edits across 1 Regions in 87 ms; skipped=0; WAL=hdfs://localhost:34445/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731595916722/wal-1, size=469, length=469, corrupted=false, cancelled=false at 1731595917128 2024-11-14T14:51:57,128 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-14T14:51:57,130 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:34445/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731595916722, archiveDir=hdfs://localhost:34445/hbase/oldWALs, maxLogs=32 2024-11-14T14:51:57,144 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testnameconflictwhensplit1-manual,16010,1731595916722/wal.1731595917131, exclude list is [], retry=0 2024-11-14T14:51:57,147 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42819,DS-c1b8ccdc-5e2c-4b3d-bb1e-6941b40dfb7e,DISK] 2024-11-14T14:51:57,148 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44549,DS-913adb4a-7946-4f46-896c-56bcdc4fcfa8,DISK] 2024-11-14T14:51:57,148 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36189,DS-5c245456-9209-4937-bfad-43a9a1b09820,DISK] 2024-11-14T14:51:57,176 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testnameconflictwhensplit1-manual,16010,1731595916722/wal.1731595917131 2024-11-14T14:51:57,177 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34777:34777),(127.0.0.1/127.0.0.1:33515:33515),(127.0.0.1/127.0.0.1:33623:33623)] 2024-11-14T14:51:57,177 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 971f94bfe32ca05e7c77cc84c8a61347, NAME => 'testReplayEditsWrittenIntoWAL,,1731595916723.971f94bfe32ca05e7c77cc84c8a61347.', STARTKEY => '', ENDKEY => ''} 2024-11-14T14:51:57,177 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1731595916723.971f94bfe32ca05e7c77cc84c8a61347.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T14:51:57,177 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 971f94bfe32ca05e7c77cc84c8a61347 2024-11-14T14:51:57,178 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 971f94bfe32ca05e7c77cc84c8a61347 2024-11-14T14:51:57,180 INFO [StoreOpener-971f94bfe32ca05e7c77cc84c8a61347-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 971f94bfe32ca05e7c77cc84c8a61347 2024-11-14T14:51:57,182 INFO [StoreOpener-971f94bfe32ca05e7c77cc84c8a61347-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 971f94bfe32ca05e7c77cc84c8a61347 columnFamilyName a 2024-11-14T14:51:57,183 DEBUG [StoreOpener-971f94bfe32ca05e7c77cc84c8a61347-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T14:51:57,183 INFO [StoreOpener-971f94bfe32ca05e7c77cc84c8a61347-1 {}] regionserver.HStore(327): Store=971f94bfe32ca05e7c77cc84c8a61347/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T14:51:57,184 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 971f94bfe32ca05e7c77cc84c8a61347 2024-11-14T14:51:57,185 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34445/hbase/data/default/testReplayEditsWrittenIntoWAL/971f94bfe32ca05e7c77cc84c8a61347 2024-11-14T14:51:57,187 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:34445/hbase/data/default/testReplayEditsWrittenIntoWAL/971f94bfe32ca05e7c77cc84c8a61347 2024-11-14T14:51:57,189 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:34445/hbase/data/default/testReplayEditsWrittenIntoWAL/971f94bfe32ca05e7c77cc84c8a61347/recovered.edits/0000000000000000002 2024-11-14T14:51:57,218 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-14T14:51:57,220 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 2, skipped 0, firstSequenceIdInLog=1, maxSequenceIdInLog=2, path=hdfs://localhost:34445/hbase/data/default/testReplayEditsWrittenIntoWAL/971f94bfe32ca05e7c77cc84c8a61347/recovered.edits/0000000000000000002 2024-11-14T14:51:57,220 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 971f94bfe32ca05e7c77cc84c8a61347 1/1 column families, dataSize=108 B heapSize=512 B 2024-11-14T14:51:57,244 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/hbase/data/default/testReplayEditsWrittenIntoWAL/971f94bfe32ca05e7c77cc84c8a61347/.tmp/a/0bc70bc68c604bbebe3134b6b624f84f is 58, key is testReplayEditsWrittenIntoWAL/a:1/1731595916816/Put/seqid=0 2024-11-14T14:51:57,253 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42819 is added to blk_1073741852_1028 (size=5170) 2024-11-14T14:51:57,253 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36189 is added to blk_1073741852_1028 (size=5170) 2024-11-14T14:51:57,254 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=108 B at sequenceid=2 (bloomFilter=true), to=hdfs://localhost:34445/hbase/data/default/testReplayEditsWrittenIntoWAL/971f94bfe32ca05e7c77cc84c8a61347/.tmp/a/0bc70bc68c604bbebe3134b6b624f84f 2024-11-14T14:51:57,254 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44549 is added to blk_1073741852_1028 (size=5170) 2024-11-14T14:51:57,262 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/hbase/data/default/testReplayEditsWrittenIntoWAL/971f94bfe32ca05e7c77cc84c8a61347/.tmp/a/0bc70bc68c604bbebe3134b6b624f84f as hdfs://localhost:34445/hbase/data/default/testReplayEditsWrittenIntoWAL/971f94bfe32ca05e7c77cc84c8a61347/a/0bc70bc68c604bbebe3134b6b624f84f 2024-11-14T14:51:57,271 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/hbase/data/default/testReplayEditsWrittenIntoWAL/971f94bfe32ca05e7c77cc84c8a61347/a/0bc70bc68c604bbebe3134b6b624f84f, entries=2, sequenceid=2, filesize=5.0 K 2024-11-14T14:51:57,271 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for 971f94bfe32ca05e7c77cc84c8a61347 in 51ms, sequenceid=2, compaction requested=false; wal=null 2024-11-14T14:51:57,272 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:34445/hbase/data/default/testReplayEditsWrittenIntoWAL/971f94bfe32ca05e7c77cc84c8a61347/recovered.edits/0000000000000000002 2024-11-14T14:51:57,273 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 971f94bfe32ca05e7c77cc84c8a61347 2024-11-14T14:51:57,273 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 971f94bfe32ca05e7c77cc84c8a61347 2024-11-14T14:51:57,275 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 971f94bfe32ca05e7c77cc84c8a61347 2024-11-14T14:51:57,278 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34445/hbase/data/default/testReplayEditsWrittenIntoWAL/971f94bfe32ca05e7c77cc84c8a61347/recovered.edits/2.seqid, newMaxSeqId=2, maxSeqId=1 2024-11-14T14:51:57,279 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 971f94bfe32ca05e7c77cc84c8a61347; next sequenceid=3; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69368453, jitterRate=0.03367049992084503}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-14T14:51:57,280 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 971f94bfe32ca05e7c77cc84c8a61347: Writing region info on filesystem at 1731595917178Initializing all the Stores at 1731595917179 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731595917179Obtaining lock to block concurrent updates at 1731595917220 (+41 ms)Preparing flush snapshotting stores in 971f94bfe32ca05e7c77cc84c8a61347 at 1731595917220Finished memstore snapshotting testReplayEditsWrittenIntoWAL,,1731595916723.971f94bfe32ca05e7c77cc84c8a61347., syncing WAL and waiting on mvcc, flushsize=dataSize=108, getHeapSize=496, getOffHeapSize=0, getCellsCount=2 at 1731595917221 (+1 ms)Flushing stores of testReplayEditsWrittenIntoWAL,,1731595916723.971f94bfe32ca05e7c77cc84c8a61347. at 1731595917221Flushing 971f94bfe32ca05e7c77cc84c8a61347/a: creating writer at 1731595917221Flushing 971f94bfe32ca05e7c77cc84c8a61347/a: appending metadata at 1731595917242 (+21 ms)Flushing 971f94bfe32ca05e7c77cc84c8a61347/a: closing flushed file at 1731595917242Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7dedaa83: reopening flushed file at 1731595917261 (+19 ms)Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for 971f94bfe32ca05e7c77cc84c8a61347 in 51ms, sequenceid=2, compaction requested=false; wal=null at 1731595917271 (+10 ms)Cleaning up temporary data from old regions at 1731595917273 (+2 ms)Region opened successfully at 1731595917280 (+7 ms) 2024-11-14T14:51:57,296 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testNameConflictWhenSplit1 Thread=387 (was 377) Potentially hanging thread: AsyncFSWAL-10-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1055376130-172.17.0.2-1731595909704:blk_1073741851_1027, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1055376130-172.17.0.2-1731595909704:blk_1073741851_1027, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1791616411_22 at /127.0.0.1:55466 [Receiving block BP-1055376130-172.17.0.2-1731595909704:blk_1073741851_1027] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1791616411_22 at /127.0.0.1:34152 [Waiting for operation #6] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1791616411_22 at /127.0.0.1:34408 [Receiving block BP-1055376130-172.17.0.2-1731595909704:blk_1073741851_1027] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1791616411_22 at /127.0.0.1:55398 [Waiting for operation #21] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1791616411_22 at /127.0.0.1:36798 [Receiving block BP-1055376130-172.17.0.2-1731595909704:blk_1073741851_1027] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-10-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1055376130-172.17.0.2-1731595909704:blk_1073741851_1027, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-10-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=785 (was 703) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=199 (was 199), ProcessCount=11 (was 11), AvailableMemoryMB=7951 (was 7973) 2024-11-14T14:51:57,308 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsWrittenIntoWAL Thread=387, OpenFileDescriptor=785, MaxFileDescriptor=1048576, SystemLoadAverage=199, ProcessCount=11, AvailableMemoryMB=7951 2024-11-14T14:51:57,324 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-14T14:51:57,327 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-14T14:51:57,328 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-14T14:51:57,331 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-57116822, suffix=, logDir=hdfs://localhost:34445/hbase/WALs/hregion-57116822, archiveDir=hdfs://localhost:34445/hbase/oldWALs, maxLogs=32 2024-11-14T14:51:57,344 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-57116822/hregion-57116822.1731595917332, exclude list is [], retry=0 2024-11-14T14:51:57,348 DEBUG [AsyncFSWAL-12-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36189,DS-5c245456-9209-4937-bfad-43a9a1b09820,DISK] 2024-11-14T14:51:57,348 DEBUG [AsyncFSWAL-12-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44549,DS-913adb4a-7946-4f46-896c-56bcdc4fcfa8,DISK] 2024-11-14T14:51:57,349 DEBUG [AsyncFSWAL-12-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42819,DS-c1b8ccdc-5e2c-4b3d-bb1e-6941b40dfb7e,DISK] 2024-11-14T14:51:57,378 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-57116822/hregion-57116822.1731595917332 2024-11-14T14:51:57,378 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33623:33623),(127.0.0.1/127.0.0.1:33515:33515),(127.0.0.1/127.0.0.1:34777:34777)] 2024-11-14T14:51:57,378 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => cb591ef09d8c39b8ddb49f80e4e9909a, NAME => 'testReplayEditsWrittenIntoWAL,,1731595917325.cb591ef09d8c39b8ddb49f80e4e9909a.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenIntoWAL', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34445/hbase 2024-11-14T14:51:57,389 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42819 is added to blk_1073741854_1030 (size=64) 2024-11-14T14:51:57,389 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36189 is added to blk_1073741854_1030 (size=64) 2024-11-14T14:51:57,390 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44549 is added to blk_1073741854_1030 (size=64) 2024-11-14T14:51:57,391 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1731595917325.cb591ef09d8c39b8ddb49f80e4e9909a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T14:51:57,392 INFO [StoreOpener-cb591ef09d8c39b8ddb49f80e4e9909a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region cb591ef09d8c39b8ddb49f80e4e9909a 2024-11-14T14:51:57,394 INFO [StoreOpener-cb591ef09d8c39b8ddb49f80e4e9909a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cb591ef09d8c39b8ddb49f80e4e9909a columnFamilyName a 2024-11-14T14:51:57,395 DEBUG [StoreOpener-cb591ef09d8c39b8ddb49f80e4e9909a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T14:51:57,395 INFO [StoreOpener-cb591ef09d8c39b8ddb49f80e4e9909a-1 {}] regionserver.HStore(327): Store=cb591ef09d8c39b8ddb49f80e4e9909a/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T14:51:57,395 INFO [StoreOpener-cb591ef09d8c39b8ddb49f80e4e9909a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region cb591ef09d8c39b8ddb49f80e4e9909a 2024-11-14T14:51:57,397 INFO [StoreOpener-cb591ef09d8c39b8ddb49f80e4e9909a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cb591ef09d8c39b8ddb49f80e4e9909a columnFamilyName b 2024-11-14T14:51:57,397 DEBUG [StoreOpener-cb591ef09d8c39b8ddb49f80e4e9909a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T14:51:57,398 INFO [StoreOpener-cb591ef09d8c39b8ddb49f80e4e9909a-1 {}] regionserver.HStore(327): Store=cb591ef09d8c39b8ddb49f80e4e9909a/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T14:51:57,398 INFO [StoreOpener-cb591ef09d8c39b8ddb49f80e4e9909a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region cb591ef09d8c39b8ddb49f80e4e9909a 2024-11-14T14:51:57,399 INFO [StoreOpener-cb591ef09d8c39b8ddb49f80e4e9909a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cb591ef09d8c39b8ddb49f80e4e9909a columnFamilyName c 2024-11-14T14:51:57,399 DEBUG [StoreOpener-cb591ef09d8c39b8ddb49f80e4e9909a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T14:51:57,400 INFO [StoreOpener-cb591ef09d8c39b8ddb49f80e4e9909a-1 {}] regionserver.HStore(327): Store=cb591ef09d8c39b8ddb49f80e4e9909a/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T14:51:57,400 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for cb591ef09d8c39b8ddb49f80e4e9909a 2024-11-14T14:51:57,401 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34445/hbase/data/default/testReplayEditsWrittenIntoWAL/cb591ef09d8c39b8ddb49f80e4e9909a 2024-11-14T14:51:57,402 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34445/hbase/data/default/testReplayEditsWrittenIntoWAL/cb591ef09d8c39b8ddb49f80e4e9909a 2024-11-14T14:51:57,403 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for cb591ef09d8c39b8ddb49f80e4e9909a 2024-11-14T14:51:57,403 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for cb591ef09d8c39b8ddb49f80e4e9909a 2024-11-14T14:51:57,404 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenIntoWAL descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-14T14:51:57,405 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for cb591ef09d8c39b8ddb49f80e4e9909a 2024-11-14T14:51:57,408 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34445/hbase/data/default/testReplayEditsWrittenIntoWAL/cb591ef09d8c39b8ddb49f80e4e9909a/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-14T14:51:57,408 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened cb591ef09d8c39b8ddb49f80e4e9909a; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61618152, jitterRate=-0.08181798458099365}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-14T14:51:57,409 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for cb591ef09d8c39b8ddb49f80e4e9909a: Writing region info on filesystem at 1731595917391Initializing all the Stores at 1731595917392 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731595917392Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731595917392Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731595917392Cleaning up temporary data from old regions at 1731595917403 (+11 ms)Region opened successfully at 1731595917409 (+6 ms) 2024-11-14T14:51:57,409 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing cb591ef09d8c39b8ddb49f80e4e9909a, disabling compactions & flushes 2024-11-14T14:51:57,409 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1731595917325.cb591ef09d8c39b8ddb49f80e4e9909a. 2024-11-14T14:51:57,409 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1731595917325.cb591ef09d8c39b8ddb49f80e4e9909a. 2024-11-14T14:51:57,409 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1731595917325.cb591ef09d8c39b8ddb49f80e4e9909a. after waiting 0 ms 2024-11-14T14:51:57,409 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1731595917325.cb591ef09d8c39b8ddb49f80e4e9909a. 2024-11-14T14:51:57,410 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1731595917325.cb591ef09d8c39b8ddb49f80e4e9909a. 2024-11-14T14:51:57,410 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for cb591ef09d8c39b8ddb49f80e4e9909a: Waiting for close lock at 1731595917409Disabling compacts and flushes for region at 1731595917409Disabling writes for close at 1731595917409Writing region close event to WAL at 1731595917409Closed at 1731595917409 2024-11-14T14:51:57,413 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36189 is added to blk_1073741853_1029 (size=175) 2024-11-14T14:51:57,413 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42819 is added to blk_1073741853_1029 (size=175) 2024-11-14T14:51:57,413 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44549 is added to blk_1073741853_1029 (size=175) 2024-11-14T14:51:57,416 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-14T14:51:57,416 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-57116822:(num 1731595917332) 2024-11-14T14:51:57,416 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-14T14:51:57,418 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:34445/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731595917323, archiveDir=hdfs://localhost:34445/hbase/oldWALs, maxLogs=32 2024-11-14T14:51:57,431 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731595917323/wal.1731595917418, exclude list is [], retry=0 2024-11-14T14:51:57,434 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42819,DS-c1b8ccdc-5e2c-4b3d-bb1e-6941b40dfb7e,DISK] 2024-11-14T14:51:57,435 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36189,DS-5c245456-9209-4937-bfad-43a9a1b09820,DISK] 2024-11-14T14:51:57,435 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44549,DS-913adb4a-7946-4f46-896c-56bcdc4fcfa8,DISK] 2024-11-14T14:51:57,474 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731595917323/wal.1731595917418 2024-11-14T14:51:57,474 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34777:34777),(127.0.0.1/127.0.0.1:33623:33623),(127.0.0.1/127.0.0.1:33515:33515)] 2024-11-14T14:51:57,705 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:34445/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731595917323/wal.1731595917418, size=0 (0bytes) 2024-11-14T14:51:57,705 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:34445/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731595917323/wal.1731595917418 might be still open, length is 0 2024-11-14T14:51:57,705 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:34445/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731595917323/wal.1731595917418 2024-11-14T14:51:57,706 WARN [IPC Server handler 0 on default port 34445 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731595917323/wal.1731595917418 has not been closed. Lease recovery is in progress. RecoveryId = 1032 for block blk_1073741855_1031 2024-11-14T14:51:57,706 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:34445/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731595917323/wal.1731595917418 after 1ms 2024-11-14T14:52:00,695 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1791616411_22 at /127.0.0.1:34436 [Receiving block BP-1055376130-172.17.0.2-1731595909704:blk_1073741855_1031] {}] datanode.DataXceiver(331): 127.0.0.1:42819:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:34436 dst: /127.0.0.1:42819 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:42819 remote=/127.0.0.1:34436]. Total timeout mills is 60000, 56977 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:52:00,696 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1791616411_22 at /127.0.0.1:36844 [Receiving block BP-1055376130-172.17.0.2-1731595909704:blk_1073741855_1031] {}] datanode.DataXceiver(331): 127.0.0.1:36189:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:36844 dst: /127.0.0.1:36189 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:52:00,696 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1791616411_22 at /127.0.0.1:55482 [Receiving block BP-1055376130-172.17.0.2-1731595909704:blk_1073741855_1031] {}] datanode.DataXceiver(331): 127.0.0.1:44549:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:55482 dst: /127.0.0.1:44549 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:52:00,714 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42819 is added to blk_1073741855_1032 (size=497775) 2024-11-14T14:52:00,716 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36189 is added to blk_1073741855_1032 (size=497775) 2024-11-14T14:52:00,719 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44549 is added to blk_1073741855_1032 (size=497775) 2024-11-14T14:52:01,009 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-14T14:52:01,060 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-14T14:52:01,708 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:34445/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731595917323/wal.1731595917418 after 4002ms 2024-11-14T14:52:01,738 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-14T14:52:01,740 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:34445/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731595917323/wal.1731595917418 took 4035ms 2024-11-14T14:52:01,745 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal.1731595917418.temp 2024-11-14T14:52:01,784 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:34445/hbase/data/default/testReplayEditsWrittenIntoWAL/cb591ef09d8c39b8ddb49f80e4e9909a/recovered.edits/0000000000000000001-wal.1731595917418.temp 2024-11-14T14:52:01,850 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:34445/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731595917323/wal.1731595917418; continuing. 2024-11-14T14:52:01,850 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:34445/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731595917323/wal.1731595917418 so closing down 2024-11-14T14:52:01,850 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-14T14:52:01,850 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-14T14:52:01,854 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44549 is added to blk_1073741856_1033 (size=497783) 2024-11-14T14:52:01,854 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42819 is added to blk_1073741856_1033 (size=497783) 2024-11-14T14:52:01,855 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36189 is added to blk_1073741856_1033 (size=497783) 2024-11-14T14:52:01,855 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:34445/hbase/data/default/testReplayEditsWrittenIntoWAL/cb591ef09d8c39b8ddb49f80e4e9909a/recovered.edits/0000000000000000001-wal.1731595917418.temp (wrote 3002 edits, skipped 0 edits in 44 ms) 2024-11-14T14:52:01,858 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:34445/hbase/data/default/testReplayEditsWrittenIntoWAL/cb591ef09d8c39b8ddb49f80e4e9909a/recovered.edits/0000000000000000001-wal.1731595917418.temp to hdfs://localhost:34445/hbase/data/default/testReplayEditsWrittenIntoWAL/cb591ef09d8c39b8ddb49f80e4e9909a/recovered.edits/0000000000000003002 2024-11-14T14:52:01,858 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 3002 edits across 1 Regions in 118 ms; skipped=0; WAL=hdfs://localhost:34445/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731595917323/wal.1731595917418, size=0, length=0, corrupted=false, cancelled=false 2024-11-14T14:52:01,858 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:34445/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731595917323/wal.1731595917418, journal: Splitting hdfs://localhost:34445/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731595917323/wal.1731595917418, size=0 (0bytes) at 1731595917705Creating recovered edits writer path=hdfs://localhost:34445/hbase/data/default/testReplayEditsWrittenIntoWAL/cb591ef09d8c39b8ddb49f80e4e9909a/recovered.edits/0000000000000000001-wal.1731595917418.temp at 1731595921784 (+4079 ms)Split 1024 edits, skipped 0 edits. at 1731595921793 (+9 ms)Split 2048 edits, skipped 0 edits. at 1731595921822 (+29 ms)Finishing writing output for hdfs://localhost:34445/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731595917323/wal.1731595917418 so closing down at 1731595921850 (+28 ms)3 split writer threads finished at 1731595921850Closed recovered edits writer path=hdfs://localhost:34445/hbase/data/default/testReplayEditsWrittenIntoWAL/cb591ef09d8c39b8ddb49f80e4e9909a/recovered.edits/0000000000000000001-wal.1731595917418.temp (wrote 3002 edits, skipped 0 edits in 44 ms) at 1731595921855 (+5 ms)Rename recovered edits hdfs://localhost:34445/hbase/data/default/testReplayEditsWrittenIntoWAL/cb591ef09d8c39b8ddb49f80e4e9909a/recovered.edits/0000000000000000001-wal.1731595917418.temp to hdfs://localhost:34445/hbase/data/default/testReplayEditsWrittenIntoWAL/cb591ef09d8c39b8ddb49f80e4e9909a/recovered.edits/0000000000000003002 at 1731595921858 (+3 ms)Processed 3002 edits across 1 Regions in 118 ms; skipped=0; WAL=hdfs://localhost:34445/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731595917323/wal.1731595917418, size=0, length=0, corrupted=false, cancelled=false at 1731595921858 2024-11-14T14:52:01,860 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:34445/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731595917323/wal.1731595917418 to hdfs://localhost:34445/hbase/oldWALs/wal.1731595917418 2024-11-14T14:52:01,862 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:34445/hbase/data/default/testReplayEditsWrittenIntoWAL/cb591ef09d8c39b8ddb49f80e4e9909a/recovered.edits/0000000000000003002 2024-11-14T14:52:01,862 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-14T14:52:01,865 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:34445/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731595917323, archiveDir=hdfs://localhost:34445/hbase/oldWALs, maxLogs=32 2024-11-14T14:52:01,882 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731595917323/wal.1731595921865, exclude list is [], retry=0 2024-11-14T14:52:01,885 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44549,DS-913adb4a-7946-4f46-896c-56bcdc4fcfa8,DISK] 2024-11-14T14:52:01,886 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42819,DS-c1b8ccdc-5e2c-4b3d-bb1e-6941b40dfb7e,DISK] 2024-11-14T14:52:01,886 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36189,DS-5c245456-9209-4937-bfad-43a9a1b09820,DISK] 2024-11-14T14:52:01,913 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731595917323/wal.1731595921865 2024-11-14T14:52:01,914 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33515:33515),(127.0.0.1/127.0.0.1:34777:34777),(127.0.0.1/127.0.0.1:33623:33623)] 2024-11-14T14:52:01,914 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1731595917325.cb591ef09d8c39b8ddb49f80e4e9909a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T14:52:01,916 INFO [StoreOpener-cb591ef09d8c39b8ddb49f80e4e9909a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region cb591ef09d8c39b8ddb49f80e4e9909a 2024-11-14T14:52:01,917 INFO [StoreOpener-cb591ef09d8c39b8ddb49f80e4e9909a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cb591ef09d8c39b8ddb49f80e4e9909a columnFamilyName a 2024-11-14T14:52:01,917 DEBUG [StoreOpener-cb591ef09d8c39b8ddb49f80e4e9909a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T14:52:01,918 INFO [StoreOpener-cb591ef09d8c39b8ddb49f80e4e9909a-1 {}] regionserver.HStore(327): Store=cb591ef09d8c39b8ddb49f80e4e9909a/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T14:52:01,918 INFO [StoreOpener-cb591ef09d8c39b8ddb49f80e4e9909a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region cb591ef09d8c39b8ddb49f80e4e9909a 2024-11-14T14:52:01,919 INFO [StoreOpener-cb591ef09d8c39b8ddb49f80e4e9909a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cb591ef09d8c39b8ddb49f80e4e9909a columnFamilyName b 2024-11-14T14:52:01,919 DEBUG [StoreOpener-cb591ef09d8c39b8ddb49f80e4e9909a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T14:52:01,919 INFO [StoreOpener-cb591ef09d8c39b8ddb49f80e4e9909a-1 {}] regionserver.HStore(327): Store=cb591ef09d8c39b8ddb49f80e4e9909a/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T14:52:01,920 INFO [StoreOpener-cb591ef09d8c39b8ddb49f80e4e9909a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region cb591ef09d8c39b8ddb49f80e4e9909a 2024-11-14T14:52:01,921 INFO [StoreOpener-cb591ef09d8c39b8ddb49f80e4e9909a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cb591ef09d8c39b8ddb49f80e4e9909a columnFamilyName c 2024-11-14T14:52:01,921 DEBUG [StoreOpener-cb591ef09d8c39b8ddb49f80e4e9909a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T14:52:01,921 INFO [StoreOpener-cb591ef09d8c39b8ddb49f80e4e9909a-1 {}] regionserver.HStore(327): Store=cb591ef09d8c39b8ddb49f80e4e9909a/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T14:52:01,921 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for cb591ef09d8c39b8ddb49f80e4e9909a 2024-11-14T14:52:01,922 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34445/hbase/data/default/testReplayEditsWrittenIntoWAL/cb591ef09d8c39b8ddb49f80e4e9909a 2024-11-14T14:52:01,924 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:34445/hbase/data/default/testReplayEditsWrittenIntoWAL/cb591ef09d8c39b8ddb49f80e4e9909a 2024-11-14T14:52:01,925 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:34445/hbase/data/default/testReplayEditsWrittenIntoWAL/cb591ef09d8c39b8ddb49f80e4e9909a/recovered.edits/0000000000000003002 2024-11-14T14:52:01,955 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-14T14:52:01,994 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-11-14T14:52:02,391 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing cb591ef09d8c39b8ddb49f80e4e9909a 3/3 column families, dataSize=42.49 KB heapSize=100.11 KB 2024-11-14T14:52:02,430 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/hbase/data/default/testReplayEditsWrittenIntoWAL/cb591ef09d8c39b8ddb49f80e4e9909a/.tmp/a/83626947f1e3404281584c8c19e9cd04 is 62, key is testReplayEditsWrittenIntoWAL/a:100/1731595917481/Put/seqid=0 2024-11-14T14:52:02,439 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42819 is added to blk_1073741858_1035 (size=50463) 2024-11-14T14:52:02,440 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36189 is added to blk_1073741858_1035 (size=50463) 2024-11-14T14:52:02,440 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44549 is added to blk_1073741858_1035 (size=50463) 2024-11-14T14:52:02,441 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=754 (bloomFilter=true), to=hdfs://localhost:34445/hbase/data/default/testReplayEditsWrittenIntoWAL/cb591ef09d8c39b8ddb49f80e4e9909a/.tmp/a/83626947f1e3404281584c8c19e9cd04 2024-11-14T14:52:02,450 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/hbase/data/default/testReplayEditsWrittenIntoWAL/cb591ef09d8c39b8ddb49f80e4e9909a/.tmp/a/83626947f1e3404281584c8c19e9cd04 as hdfs://localhost:34445/hbase/data/default/testReplayEditsWrittenIntoWAL/cb591ef09d8c39b8ddb49f80e4e9909a/a/83626947f1e3404281584c8c19e9cd04 2024-11-14T14:52:02,458 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/hbase/data/default/testReplayEditsWrittenIntoWAL/cb591ef09d8c39b8ddb49f80e4e9909a/a/83626947f1e3404281584c8c19e9cd04, entries=754, sequenceid=754, filesize=49.3 K 2024-11-14T14:52:02,459 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~42.49 KB/43512, heapSize ~99.59 KB/101984, currentSize=0 B/0 for cb591ef09d8c39b8ddb49f80e4e9909a in 68ms, sequenceid=754, compaction requested=false; wal=null 2024-11-14T14:52:02,476 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-11-14T14:52:02,477 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing cb591ef09d8c39b8ddb49f80e4e9909a 3/3 column families, dataSize=42.49 KB heapSize=100.11 KB 2024-11-14T14:52:02,485 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/hbase/data/default/testReplayEditsWrittenIntoWAL/cb591ef09d8c39b8ddb49f80e4e9909a/.tmp/a/635409060ce242148735272c0b83daae is 62, key is testReplayEditsWrittenIntoWAL/a:754/1731595917531/Put/seqid=0 2024-11-14T14:52:02,495 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42819 is added to blk_1073741859_1036 (size=20072) 2024-11-14T14:52:02,495 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36189 is added to blk_1073741859_1036 (size=20072) 2024-11-14T14:52:02,497 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44549 is added to blk_1073741859_1036 (size=20072) 2024-11-14T14:52:02,497 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.93 KB at sequenceid=1508 (bloomFilter=true), to=hdfs://localhost:34445/hbase/data/default/testReplayEditsWrittenIntoWAL/cb591ef09d8c39b8ddb49f80e4e9909a/.tmp/a/635409060ce242148735272c0b83daae 2024-11-14T14:52:02,527 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/hbase/data/default/testReplayEditsWrittenIntoWAL/cb591ef09d8c39b8ddb49f80e4e9909a/.tmp/b/6e279958337c42bc943ee54b8f10db01 is 62, key is testReplayEditsWrittenIntoWAL/b:100/1731595917554/Put/seqid=0 2024-11-14T14:52:02,536 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44549 is added to blk_1073741860_1037 (size=35835) 2024-11-14T14:52:02,536 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36189 is added to blk_1073741860_1037 (size=35835) 2024-11-14T14:52:02,536 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42819 is added to blk_1073741860_1037 (size=35835) 2024-11-14T14:52:02,537 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=28.56 KB at sequenceid=1508 (bloomFilter=true), to=hdfs://localhost:34445/hbase/data/default/testReplayEditsWrittenIntoWAL/cb591ef09d8c39b8ddb49f80e4e9909a/.tmp/b/6e279958337c42bc943ee54b8f10db01 2024-11-14T14:52:02,546 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/hbase/data/default/testReplayEditsWrittenIntoWAL/cb591ef09d8c39b8ddb49f80e4e9909a/.tmp/a/635409060ce242148735272c0b83daae as hdfs://localhost:34445/hbase/data/default/testReplayEditsWrittenIntoWAL/cb591ef09d8c39b8ddb49f80e4e9909a/a/635409060ce242148735272c0b83daae 2024-11-14T14:52:02,553 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/hbase/data/default/testReplayEditsWrittenIntoWAL/cb591ef09d8c39b8ddb49f80e4e9909a/a/635409060ce242148735272c0b83daae, entries=246, sequenceid=1508, filesize=19.6 K 2024-11-14T14:52:02,554 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/hbase/data/default/testReplayEditsWrittenIntoWAL/cb591ef09d8c39b8ddb49f80e4e9909a/.tmp/b/6e279958337c42bc943ee54b8f10db01 as hdfs://localhost:34445/hbase/data/default/testReplayEditsWrittenIntoWAL/cb591ef09d8c39b8ddb49f80e4e9909a/b/6e279958337c42bc943ee54b8f10db01 2024-11-14T14:52:02,561 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/hbase/data/default/testReplayEditsWrittenIntoWAL/cb591ef09d8c39b8ddb49f80e4e9909a/b/6e279958337c42bc943ee54b8f10db01, entries=508, sequenceid=1508, filesize=35.0 K 2024-11-14T14:52:02,562 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~42.49 KB/43512, heapSize ~99.83 KB/102224, currentSize=0 B/0 for cb591ef09d8c39b8ddb49f80e4e9909a in 84ms, sequenceid=1508, compaction requested=false; wal=null 2024-11-14T14:52:02,579 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-11-14T14:52:02,579 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing cb591ef09d8c39b8ddb49f80e4e9909a 3/3 column families, dataSize=42.49 KB heapSize=100.11 KB 2024-11-14T14:52:02,589 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/hbase/data/default/testReplayEditsWrittenIntoWAL/cb591ef09d8c39b8ddb49f80e4e9909a/.tmp/b/a519f04b1ccb44308faa228d39a85a0f is 62, key is testReplayEditsWrittenIntoWAL/b:508/1731595917580/Put/seqid=0 2024-11-14T14:52:02,598 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42819 is added to blk_1073741861_1038 (size=35082) 2024-11-14T14:52:02,598 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36189 is added to blk_1073741861_1038 (size=35082) 2024-11-14T14:52:02,599 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44549 is added to blk_1073741861_1038 (size=35082) 2024-11-14T14:52:02,599 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=27.87 KB at sequenceid=2262 (bloomFilter=true), to=hdfs://localhost:34445/hbase/data/default/testReplayEditsWrittenIntoWAL/cb591ef09d8c39b8ddb49f80e4e9909a/.tmp/b/a519f04b1ccb44308faa228d39a85a0f 2024-11-14T14:52:02,633 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/hbase/data/default/testReplayEditsWrittenIntoWAL/cb591ef09d8c39b8ddb49f80e4e9909a/.tmp/c/97aab40faf71419bb1d508013e9afa92 is 62, key is testReplayEditsWrittenIntoWAL/c:100/1731595917618/Put/seqid=0 2024-11-14T14:52:02,642 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42819 is added to blk_1073741862_1039 (size=20825) 2024-11-14T14:52:02,642 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44549 is added to blk_1073741862_1039 (size=20825) 2024-11-14T14:52:02,643 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36189 is added to blk_1073741862_1039 (size=20825) 2024-11-14T14:52:02,644 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=14.63 KB at sequenceid=2262 (bloomFilter=true), to=hdfs://localhost:34445/hbase/data/default/testReplayEditsWrittenIntoWAL/cb591ef09d8c39b8ddb49f80e4e9909a/.tmp/c/97aab40faf71419bb1d508013e9afa92 2024-11-14T14:52:02,653 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/hbase/data/default/testReplayEditsWrittenIntoWAL/cb591ef09d8c39b8ddb49f80e4e9909a/.tmp/b/a519f04b1ccb44308faa228d39a85a0f as hdfs://localhost:34445/hbase/data/default/testReplayEditsWrittenIntoWAL/cb591ef09d8c39b8ddb49f80e4e9909a/b/a519f04b1ccb44308faa228d39a85a0f 2024-11-14T14:52:02,660 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/hbase/data/default/testReplayEditsWrittenIntoWAL/cb591ef09d8c39b8ddb49f80e4e9909a/b/a519f04b1ccb44308faa228d39a85a0f, entries=492, sequenceid=2262, filesize=34.3 K 2024-11-14T14:52:02,662 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/hbase/data/default/testReplayEditsWrittenIntoWAL/cb591ef09d8c39b8ddb49f80e4e9909a/.tmp/c/97aab40faf71419bb1d508013e9afa92 as hdfs://localhost:34445/hbase/data/default/testReplayEditsWrittenIntoWAL/cb591ef09d8c39b8ddb49f80e4e9909a/c/97aab40faf71419bb1d508013e9afa92 2024-11-14T14:52:02,669 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/hbase/data/default/testReplayEditsWrittenIntoWAL/cb591ef09d8c39b8ddb49f80e4e9909a/c/97aab40faf71419bb1d508013e9afa92, entries=262, sequenceid=2262, filesize=20.3 K 2024-11-14T14:52:02,669 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~42.49 KB/43512, heapSize ~99.83 KB/102224, currentSize=0 B/0 for cb591ef09d8c39b8ddb49f80e4e9909a in 90ms, sequenceid=2262, compaction requested=false; wal=null 2024-11-14T14:52:02,679 WARN [Time-limited test {}] regionserver.HRegion(5722): No family for cell testReplayEditsWrittenIntoWAL/another family:testReplayEditsWrittenIntoWAL/1731595917670/Put/vlen=29/seqid=0 in region testReplayEditsWrittenIntoWAL,,1731595917325.cb591ef09d8c39b8ddb49f80e4e9909a. 2024-11-14T14:52:02,682 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 3001, skipped 1, firstSequenceIdInLog=1, maxSequenceIdInLog=3002, path=hdfs://localhost:34445/hbase/data/default/testReplayEditsWrittenIntoWAL/cb591ef09d8c39b8ddb49f80e4e9909a/recovered.edits/0000000000000003002 2024-11-14T14:52:02,683 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-11-14T14:52:02,683 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing cb591ef09d8c39b8ddb49f80e4e9909a 3/3 column families, dataSize=41.85 KB heapSize=98.89 KB 2024-11-14T14:52:02,691 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/hbase/data/default/testReplayEditsWrittenIntoWAL/cb591ef09d8c39b8ddb49f80e4e9909a/.tmp/c/227ae8cff2aa48459cd53f63194b9aef is 62, key is testReplayEditsWrittenIntoWAL/c:262/1731595917625/Put/seqid=0 2024-11-14T14:52:02,699 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36189 is added to blk_1073741863_1040 (size=50301) 2024-11-14T14:52:02,700 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44549 is added to blk_1073741863_1040 (size=50301) 2024-11-14T14:52:02,701 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42819 is added to blk_1073741863_1040 (size=50301) 2024-11-14T14:52:02,701 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=41.85 KB at sequenceid=3002 (bloomFilter=true), to=hdfs://localhost:34445/hbase/data/default/testReplayEditsWrittenIntoWAL/cb591ef09d8c39b8ddb49f80e4e9909a/.tmp/c/227ae8cff2aa48459cd53f63194b9aef 2024-11-14T14:52:02,709 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 227ae8cff2aa48459cd53f63194b9aef 2024-11-14T14:52:02,710 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/hbase/data/default/testReplayEditsWrittenIntoWAL/cb591ef09d8c39b8ddb49f80e4e9909a/.tmp/c/227ae8cff2aa48459cd53f63194b9aef as hdfs://localhost:34445/hbase/data/default/testReplayEditsWrittenIntoWAL/cb591ef09d8c39b8ddb49f80e4e9909a/c/227ae8cff2aa48459cd53f63194b9aef 2024-11-14T14:52:02,717 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 227ae8cff2aa48459cd53f63194b9aef 2024-11-14T14:52:02,718 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/hbase/data/default/testReplayEditsWrittenIntoWAL/cb591ef09d8c39b8ddb49f80e4e9909a/c/227ae8cff2aa48459cd53f63194b9aef, entries=739, sequenceid=3002, filesize=49.1 K 2024-11-14T14:52:02,718 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~41.85 KB/42854, heapSize ~98.38 KB/100736, currentSize=0 B/0 for cb591ef09d8c39b8ddb49f80e4e9909a in 35ms, sequenceid=3002, compaction requested=false; wal=null 2024-11-14T14:52:02,719 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:34445/hbase/data/default/testReplayEditsWrittenIntoWAL/cb591ef09d8c39b8ddb49f80e4e9909a/recovered.edits/0000000000000003002 2024-11-14T14:52:02,720 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for cb591ef09d8c39b8ddb49f80e4e9909a 2024-11-14T14:52:02,720 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for cb591ef09d8c39b8ddb49f80e4e9909a 2024-11-14T14:52:02,721 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenIntoWAL descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-14T14:52:02,723 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for cb591ef09d8c39b8ddb49f80e4e9909a 2024-11-14T14:52:02,726 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34445/hbase/data/default/testReplayEditsWrittenIntoWAL/cb591ef09d8c39b8ddb49f80e4e9909a/recovered.edits/3002.seqid, newMaxSeqId=3002, maxSeqId=1 2024-11-14T14:52:02,727 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened cb591ef09d8c39b8ddb49f80e4e9909a; next sequenceid=3003; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=204800, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59238392, jitterRate=-0.11727917194366455}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-14T14:52:02,727 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for cb591ef09d8c39b8ddb49f80e4e9909a: Writing region info on filesystem at 1731595921914Initializing all the Stores at 1731595921916 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731595921916Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731595921916Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731595921916Cleaning up temporary data from old regions at 1731595922720 (+804 ms)Region opened successfully at 1731595922727 (+7 ms) 2024-11-14T14:52:02,801 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing cb591ef09d8c39b8ddb49f80e4e9909a, disabling compactions & flushes 2024-11-14T14:52:02,801 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1731595917325.cb591ef09d8c39b8ddb49f80e4e9909a. 2024-11-14T14:52:02,801 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1731595917325.cb591ef09d8c39b8ddb49f80e4e9909a. 2024-11-14T14:52:02,801 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1731595917325.cb591ef09d8c39b8ddb49f80e4e9909a. after waiting 0 ms 2024-11-14T14:52:02,801 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1731595917325.cb591ef09d8c39b8ddb49f80e4e9909a. 2024-11-14T14:52:02,803 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1731595917325.cb591ef09d8c39b8ddb49f80e4e9909a. 2024-11-14T14:52:02,804 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for cb591ef09d8c39b8ddb49f80e4e9909a: Waiting for close lock at 1731595922801Disabling compacts and flushes for region at 1731595922801Disabling writes for close at 1731595922801Writing region close event to WAL at 1731595922803 (+2 ms)Closed at 1731595922803 2024-11-14T14:52:02,807 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44549 is added to blk_1073741857_1034 (size=175) 2024-11-14T14:52:02,808 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42819 is added to blk_1073741857_1034 (size=175) 2024-11-14T14:52:02,808 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36189 is added to blk_1073741857_1034 (size=175) 2024-11-14T14:52:02,812 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-14T14:52:02,812 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1731595921865) 2024-11-14T14:52:02,832 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsWrittenIntoWAL Thread=403 (was 387) Potentially hanging thread: IPC Client (1319272049) connection to localhost/127.0.0.1:34445 from jenkins.replay.wal.secondtime java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c843459-ed19-f974-f761-d73dee21a91b/cluster_fceed6e6-d3b5-652e-82f8-dddc2c9b7e2b/data/data5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkins.replay.wal.secondtime@localhost:34445 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: AsyncFSWAL-12-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-12-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:39351 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c843459-ed19-f974-f761-d73dee21a91b/cluster_fceed6e6-d3b5-652e-82f8-dddc2c9b7e2b/data/data6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1319272049) connection to localhost/127.0.0.1:39351 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c843459-ed19-f974-f761-d73dee21a91b/cluster_fceed6e6-d3b5-652e-82f8-dddc2c9b7e2b/data/data2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:34459 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1319272049) connection to localhost/127.0.0.1:34459 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1900288624_22 at /127.0.0.1:33652 [Waiting for operation #3] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1900288624_22 at /127.0.0.1:57792 [Waiting for operation #17] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-12-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1900288624_22 at /127.0.0.1:33684 [Waiting for operation #3] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c843459-ed19-f974-f761-d73dee21a91b/cluster_fceed6e6-d3b5-652e-82f8-dddc2c9b7e2b/data/data1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1900288624_22 at /127.0.0.1:53112 [Waiting for operation #27] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=865 (was 785) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=183 (was 199), ProcessCount=11 (was 11), AvailableMemoryMB=7836 (was 7951) 2024-11-14T14:52:02,847 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#test2727 Thread=403, OpenFileDescriptor=865, MaxFileDescriptor=1048576, SystemLoadAverage=183, ProcessCount=11, AvailableMemoryMB=7835 2024-11-14T14:52:02,864 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-14T14:52:02,867 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-14T14:52:02,868 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-14T14:52:02,871 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-01102891, suffix=, logDir=hdfs://localhost:34445/hbase/WALs/hregion-01102891, archiveDir=hdfs://localhost:34445/hbase/oldWALs, maxLogs=32 2024-11-14T14:52:02,886 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-01102891/hregion-01102891.1731595922872, exclude list is [], retry=0 2024-11-14T14:52:02,889 DEBUG [AsyncFSWAL-14-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42819,DS-c1b8ccdc-5e2c-4b3d-bb1e-6941b40dfb7e,DISK] 2024-11-14T14:52:02,890 DEBUG [AsyncFSWAL-14-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44549,DS-913adb4a-7946-4f46-896c-56bcdc4fcfa8,DISK] 2024-11-14T14:52:02,890 DEBUG [AsyncFSWAL-14-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36189,DS-5c245456-9209-4937-bfad-43a9a1b09820,DISK] 2024-11-14T14:52:02,921 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-01102891/hregion-01102891.1731595922872 2024-11-14T14:52:02,921 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34777:34777),(127.0.0.1/127.0.0.1:33515:33515),(127.0.0.1/127.0.0.1:33623:33623)] 2024-11-14T14:52:02,921 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 340f74ab36b3e883ad7182d7bbc26926, NAME => 'test2727,,1731595922865.340f74ab36b3e883ad7182d7bbc26926.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='test2727', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34445/hbase 2024-11-14T14:52:02,936 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42819 is added to blk_1073741865_1042 (size=43) 2024-11-14T14:52:02,936 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36189 is added to blk_1073741865_1042 (size=43) 2024-11-14T14:52:02,939 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44549 is added to blk_1073741865_1042 (size=43) 2024-11-14T14:52:02,939 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated test2727,,1731595922865.340f74ab36b3e883ad7182d7bbc26926.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T14:52:02,941 INFO [StoreOpener-340f74ab36b3e883ad7182d7bbc26926-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 340f74ab36b3e883ad7182d7bbc26926 2024-11-14T14:52:02,943 INFO [StoreOpener-340f74ab36b3e883ad7182d7bbc26926-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 340f74ab36b3e883ad7182d7bbc26926 columnFamilyName a 2024-11-14T14:52:02,943 DEBUG [StoreOpener-340f74ab36b3e883ad7182d7bbc26926-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T14:52:02,943 INFO [StoreOpener-340f74ab36b3e883ad7182d7bbc26926-1 {}] regionserver.HStore(327): Store=340f74ab36b3e883ad7182d7bbc26926/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T14:52:02,944 INFO [StoreOpener-340f74ab36b3e883ad7182d7bbc26926-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 340f74ab36b3e883ad7182d7bbc26926 2024-11-14T14:52:02,945 INFO [StoreOpener-340f74ab36b3e883ad7182d7bbc26926-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 340f74ab36b3e883ad7182d7bbc26926 columnFamilyName b 2024-11-14T14:52:02,945 DEBUG [StoreOpener-340f74ab36b3e883ad7182d7bbc26926-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T14:52:02,946 INFO [StoreOpener-340f74ab36b3e883ad7182d7bbc26926-1 {}] regionserver.HStore(327): Store=340f74ab36b3e883ad7182d7bbc26926/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T14:52:02,946 INFO [StoreOpener-340f74ab36b3e883ad7182d7bbc26926-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 340f74ab36b3e883ad7182d7bbc26926 2024-11-14T14:52:02,947 INFO [StoreOpener-340f74ab36b3e883ad7182d7bbc26926-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 340f74ab36b3e883ad7182d7bbc26926 columnFamilyName c 2024-11-14T14:52:02,947 DEBUG [StoreOpener-340f74ab36b3e883ad7182d7bbc26926-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T14:52:02,948 INFO [StoreOpener-340f74ab36b3e883ad7182d7bbc26926-1 {}] regionserver.HStore(327): Store=340f74ab36b3e883ad7182d7bbc26926/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T14:52:02,948 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 340f74ab36b3e883ad7182d7bbc26926 2024-11-14T14:52:02,949 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34445/hbase/data/default/test2727/340f74ab36b3e883ad7182d7bbc26926 2024-11-14T14:52:02,949 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34445/hbase/data/default/test2727/340f74ab36b3e883ad7182d7bbc26926 2024-11-14T14:52:02,951 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 340f74ab36b3e883ad7182d7bbc26926 2024-11-14T14:52:02,951 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 340f74ab36b3e883ad7182d7bbc26926 2024-11-14T14:52:02,951 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table test2727 descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-14T14:52:02,953 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 340f74ab36b3e883ad7182d7bbc26926 2024-11-14T14:52:02,956 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34445/hbase/data/default/test2727/340f74ab36b3e883ad7182d7bbc26926/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-14T14:52:02,956 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 340f74ab36b3e883ad7182d7bbc26926; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62122648, jitterRate=-0.07430040836334229}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-14T14:52:02,958 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 340f74ab36b3e883ad7182d7bbc26926: Writing region info on filesystem at 1731595922940Initializing all the Stores at 1731595922941 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731595922941Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731595922941Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731595922941Cleaning up temporary data from old regions at 1731595922951 (+10 ms)Region opened successfully at 1731595922958 (+7 ms) 2024-11-14T14:52:02,959 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 340f74ab36b3e883ad7182d7bbc26926, disabling compactions & flushes 2024-11-14T14:52:02,959 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region test2727,,1731595922865.340f74ab36b3e883ad7182d7bbc26926. 2024-11-14T14:52:02,959 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on test2727,,1731595922865.340f74ab36b3e883ad7182d7bbc26926. 2024-11-14T14:52:02,959 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on test2727,,1731595922865.340f74ab36b3e883ad7182d7bbc26926. after waiting 0 ms 2024-11-14T14:52:02,959 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region test2727,,1731595922865.340f74ab36b3e883ad7182d7bbc26926. 2024-11-14T14:52:02,959 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed test2727,,1731595922865.340f74ab36b3e883ad7182d7bbc26926. 2024-11-14T14:52:02,960 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 340f74ab36b3e883ad7182d7bbc26926: Waiting for close lock at 1731595922959Disabling compacts and flushes for region at 1731595922959Disabling writes for close at 1731595922959Writing region close event to WAL at 1731595922959Closed at 1731595922959 2024-11-14T14:52:02,964 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44549 is added to blk_1073741864_1041 (size=175) 2024-11-14T14:52:02,964 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /hbase/WALs/hregion-01102891/hregion-01102891.1731595922872 not finished, retry = 0 2024-11-14T14:52:02,964 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36189 is added to blk_1073741864_1041 (size=175) 2024-11-14T14:52:02,964 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42819 is added to blk_1073741864_1041 (size=175) 2024-11-14T14:52:03,067 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-14T14:52:03,067 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-01102891:(num 1731595922872) 2024-11-14T14:52:03,067 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-14T14:52:03,070 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:34445/hbase/WALs/test2727-manual,16010,1731595922863, archiveDir=hdfs://localhost:34445/hbase/oldWALs, maxLogs=32 2024-11-14T14:52:03,085 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/test2727-manual,16010,1731595922863/wal.1731595923071, exclude list is [], retry=0 2024-11-14T14:52:03,088 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36189,DS-5c245456-9209-4937-bfad-43a9a1b09820,DISK] 2024-11-14T14:52:03,089 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42819,DS-c1b8ccdc-5e2c-4b3d-bb1e-6941b40dfb7e,DISK] 2024-11-14T14:52:03,089 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44549,DS-913adb4a-7946-4f46-896c-56bcdc4fcfa8,DISK] 2024-11-14T14:52:03,119 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/test2727-manual,16010,1731595922863/wal.1731595923071 2024-11-14T14:52:03,119 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33623:33623),(127.0.0.1/127.0.0.1:34777:34777),(127.0.0.1/127.0.0.1:33515:33515)] 2024-11-14T14:52:03,153 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-14T14:52:03,154 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-14T14:52:03,156 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-14T14:52:03,156 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-14T14:52:03,157 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-14T14:52:03,157 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-14T14:52:03,158 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenIntoWAL 2024-11-14T14:52:03,158 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenIntoWAL Metrics about Tables on a single HBase RegionServer 2024-11-14T14:52:03,278 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44549 is added to blk_1073741866_1043 (size=371388) 2024-11-14T14:52:03,279 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36189 is added to blk_1073741866_1043 (size=371388) 2024-11-14T14:52:03,280 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42819 is added to blk_1073741866_1043 (size=371388) 2024-11-14T14:52:03,307 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:34445/hbase/WALs/test2727-manual,16010,1731595922863/wal.1731595923071, size=362.7 K (371388bytes) 2024-11-14T14:52:03,307 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:34445/hbase/WALs/test2727-manual,16010,1731595922863/wal.1731595923071 2024-11-14T14:52:03,307 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:34445/hbase/WALs/test2727-manual,16010,1731595922863/wal.1731595923071 after 0ms 2024-11-14T14:52:03,339 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-14T14:52:03,341 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:34445/hbase/WALs/test2727-manual,16010,1731595922863/wal.1731595923071 took 35ms 2024-11-14T14:52:03,347 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal.1731595923071.temp 2024-11-14T14:52:03,386 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:34445/hbase/WALs/test2727-manual,16010,1731595922863/wal.1731595923071 so closing down 2024-11-14T14:52:03,386 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-14T14:52:03,389 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:34445/hbase/data/default/test2727/340f74ab36b3e883ad7182d7bbc26926/recovered.edits/0000000000000000001-wal.1731595923071.temp 2024-11-14T14:52:03,404 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-14T14:52:03,408 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44549 is added to blk_1073741867_1044 (size=371388) 2024-11-14T14:52:03,409 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36189 is added to blk_1073741867_1044 (size=371388) 2024-11-14T14:52:03,409 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42819 is added to blk_1073741867_1044 (size=371388) 2024-11-14T14:52:03,410 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:34445/hbase/data/default/test2727/340f74ab36b3e883ad7182d7bbc26926/recovered.edits/0000000000000000001-wal.1731595923071.temp (wrote 3000 edits, skipped 0 edits in 14 ms) 2024-11-14T14:52:03,412 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:34445/hbase/data/default/test2727/340f74ab36b3e883ad7182d7bbc26926/recovered.edits/0000000000000000001-wal.1731595923071.temp to hdfs://localhost:34445/hbase/data/default/test2727/340f74ab36b3e883ad7182d7bbc26926/recovered.edits/0000000000000003000 2024-11-14T14:52:03,412 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 3000 edits across 1 Regions in 71 ms; skipped=0; WAL=hdfs://localhost:34445/hbase/WALs/test2727-manual,16010,1731595922863/wal.1731595923071, size=362.7 K, length=371388, corrupted=false, cancelled=false 2024-11-14T14:52:03,412 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:34445/hbase/WALs/test2727-manual,16010,1731595922863/wal.1731595923071, journal: Splitting hdfs://localhost:34445/hbase/WALs/test2727-manual,16010,1731595922863/wal.1731595923071, size=362.7 K (371388bytes) at 1731595923307Split 1024 edits, skipped 0 edits. at 1731595923357 (+50 ms)Split 2048 edits, skipped 0 edits. at 1731595923372 (+15 ms)Finishing writing output for hdfs://localhost:34445/hbase/WALs/test2727-manual,16010,1731595922863/wal.1731595923071 so closing down at 1731595923386 (+14 ms)Creating recovered edits writer path=hdfs://localhost:34445/hbase/data/default/test2727/340f74ab36b3e883ad7182d7bbc26926/recovered.edits/0000000000000000001-wal.1731595923071.temp at 1731595923389 (+3 ms)3 split writer threads finished at 1731595923404 (+15 ms)Closed recovered edits writer path=hdfs://localhost:34445/hbase/data/default/test2727/340f74ab36b3e883ad7182d7bbc26926/recovered.edits/0000000000000000001-wal.1731595923071.temp (wrote 3000 edits, skipped 0 edits in 14 ms) at 1731595923410 (+6 ms)Rename recovered edits hdfs://localhost:34445/hbase/data/default/test2727/340f74ab36b3e883ad7182d7bbc26926/recovered.edits/0000000000000000001-wal.1731595923071.temp to hdfs://localhost:34445/hbase/data/default/test2727/340f74ab36b3e883ad7182d7bbc26926/recovered.edits/0000000000000003000 at 1731595923412 (+2 ms)Processed 3000 edits across 1 Regions in 71 ms; skipped=0; WAL=hdfs://localhost:34445/hbase/WALs/test2727-manual,16010,1731595922863/wal.1731595923071, size=362.7 K, length=371388, corrupted=false, cancelled=false at 1731595923412 2024-11-14T14:52:03,414 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:34445/hbase/WALs/test2727-manual,16010,1731595922863/wal.1731595923071 to hdfs://localhost:34445/hbase/oldWALs/wal.1731595923071 2024-11-14T14:52:03,415 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:34445/hbase/data/default/test2727/340f74ab36b3e883ad7182d7bbc26926/recovered.edits/0000000000000003000 2024-11-14T14:52:03,416 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-14T14:52:03,418 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:34445/hbase/WALs/test2727-manual,16010,1731595922863, archiveDir=hdfs://localhost:34445/hbase/oldWALs, maxLogs=32 2024-11-14T14:52:03,433 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/test2727-manual,16010,1731595922863/wal.1731595923418, exclude list is [], retry=0 2024-11-14T14:52:03,436 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36189,DS-5c245456-9209-4937-bfad-43a9a1b09820,DISK] 2024-11-14T14:52:03,437 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42819,DS-c1b8ccdc-5e2c-4b3d-bb1e-6941b40dfb7e,DISK] 2024-11-14T14:52:03,437 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44549,DS-913adb4a-7946-4f46-896c-56bcdc4fcfa8,DISK] 2024-11-14T14:52:03,467 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/test2727-manual,16010,1731595922863/wal.1731595923418 2024-11-14T14:52:03,467 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33623:33623),(127.0.0.1/127.0.0.1:34777:34777),(127.0.0.1/127.0.0.1:33515:33515)] 2024-11-14T14:52:03,699 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42819 is added to blk_1073741868_1045 (size=371515) 2024-11-14T14:52:03,699 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44549 is added to blk_1073741868_1045 (size=371515) 2024-11-14T14:52:03,700 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36189 is added to blk_1073741868_1045 (size=371515) 2024-11-14T14:52:03,727 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:34445/hbase/WALs/test2727-manual,16010,1731595922863/wal.1731595923418, size=362.8 K (371515bytes) 2024-11-14T14:52:03,727 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:34445/hbase/WALs/test2727-manual,16010,1731595922863/wal.1731595923418 2024-11-14T14:52:03,728 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:34445/hbase/WALs/test2727-manual,16010,1731595922863/wal.1731595923418 after 1ms 2024-11-14T14:52:03,758 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-14T14:52:03,759 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:34445/hbase/WALs/test2727-manual,16010,1731595922863/wal.1731595923418 took 32ms 2024-11-14T14:52:03,764 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000003001-wal.1731595923418.temp 2024-11-14T14:52:03,798 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:34445/hbase/WALs/test2727-manual,16010,1731595922863/wal.1731595923418 so closing down 2024-11-14T14:52:03,798 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-14T14:52:03,803 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:34445/hbase/data/default/test2727/340f74ab36b3e883ad7182d7bbc26926/recovered.edits/0000000000000003001-wal.1731595923418.temp 2024-11-14T14:52:03,813 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-14T14:52:03,819 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36189 is added to blk_1073741869_1046 (size=371515) 2024-11-14T14:52:03,819 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44549 is added to blk_1073741869_1046 (size=371515) 2024-11-14T14:52:03,819 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42819 is added to blk_1073741869_1046 (size=371515) 2024-11-14T14:52:03,820 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:34445/hbase/data/default/test2727/340f74ab36b3e883ad7182d7bbc26926/recovered.edits/0000000000000003001-wal.1731595923418.temp (wrote 3000 edits, skipped 0 edits in 10 ms) 2024-11-14T14:52:03,822 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:34445/hbase/data/default/test2727/340f74ab36b3e883ad7182d7bbc26926/recovered.edits/0000000000000003001-wal.1731595923418.temp to hdfs://localhost:34445/hbase/data/default/test2727/340f74ab36b3e883ad7182d7bbc26926/recovered.edits/0000000000000006000 2024-11-14T14:52:03,823 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 3000 edits across 1 Regions in 63 ms; skipped=0; WAL=hdfs://localhost:34445/hbase/WALs/test2727-manual,16010,1731595922863/wal.1731595923418, size=362.8 K, length=371515, corrupted=false, cancelled=false 2024-11-14T14:52:03,823 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:34445/hbase/WALs/test2727-manual,16010,1731595922863/wal.1731595923418, journal: Splitting hdfs://localhost:34445/hbase/WALs/test2727-manual,16010,1731595922863/wal.1731595923418, size=362.8 K (371515bytes) at 1731595923727Split 1024 edits, skipped 0 edits. at 1731595923773 (+46 ms)Split 2048 edits, skipped 0 edits. at 1731595923787 (+14 ms)Finishing writing output for hdfs://localhost:34445/hbase/WALs/test2727-manual,16010,1731595922863/wal.1731595923418 so closing down at 1731595923798 (+11 ms)Creating recovered edits writer path=hdfs://localhost:34445/hbase/data/default/test2727/340f74ab36b3e883ad7182d7bbc26926/recovered.edits/0000000000000003001-wal.1731595923418.temp at 1731595923803 (+5 ms)3 split writer threads finished at 1731595923813 (+10 ms)Closed recovered edits writer path=hdfs://localhost:34445/hbase/data/default/test2727/340f74ab36b3e883ad7182d7bbc26926/recovered.edits/0000000000000003001-wal.1731595923418.temp (wrote 3000 edits, skipped 0 edits in 10 ms) at 1731595923820 (+7 ms)Rename recovered edits hdfs://localhost:34445/hbase/data/default/test2727/340f74ab36b3e883ad7182d7bbc26926/recovered.edits/0000000000000003001-wal.1731595923418.temp to hdfs://localhost:34445/hbase/data/default/test2727/340f74ab36b3e883ad7182d7bbc26926/recovered.edits/0000000000000006000 at 1731595923822 (+2 ms)Processed 3000 edits across 1 Regions in 63 ms; skipped=0; WAL=hdfs://localhost:34445/hbase/WALs/test2727-manual,16010,1731595922863/wal.1731595923418, size=362.8 K, length=371515, corrupted=false, cancelled=false at 1731595923823 (+1 ms) 2024-11-14T14:52:03,825 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:34445/hbase/WALs/test2727-manual,16010,1731595922863/wal.1731595923418 to hdfs://localhost:34445/hbase/oldWALs/wal.1731595923418 2024-11-14T14:52:03,826 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:34445/hbase/data/default/test2727/340f74ab36b3e883ad7182d7bbc26926/recovered.edits/0000000000000006000 2024-11-14T14:52:03,826 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-14T14:52:03,829 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:34445/hbase/WALs/test2727-manual,16010,1731595922863, archiveDir=hdfs://localhost:34445/hbase/oldWALs, maxLogs=32 2024-11-14T14:52:03,850 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/test2727-manual,16010,1731595922863/wal.1731595923829, exclude list is [], retry=0 2024-11-14T14:52:03,854 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42819,DS-c1b8ccdc-5e2c-4b3d-bb1e-6941b40dfb7e,DISK] 2024-11-14T14:52:03,854 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44549,DS-913adb4a-7946-4f46-896c-56bcdc4fcfa8,DISK] 2024-11-14T14:52:03,855 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36189,DS-5c245456-9209-4937-bfad-43a9a1b09820,DISK] 2024-11-14T14:52:03,883 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/test2727-manual,16010,1731595922863/wal.1731595923829 2024-11-14T14:52:03,883 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34777:34777),(127.0.0.1/127.0.0.1:33515:33515),(127.0.0.1/127.0.0.1:33623:33623)] 2024-11-14T14:52:03,883 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 340f74ab36b3e883ad7182d7bbc26926, NAME => 'test2727,,1731595922865.340f74ab36b3e883ad7182d7bbc26926.', STARTKEY => '', ENDKEY => ''} 2024-11-14T14:52:03,884 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated test2727,,1731595922865.340f74ab36b3e883ad7182d7bbc26926.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T14:52:03,884 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 340f74ab36b3e883ad7182d7bbc26926 2024-11-14T14:52:03,884 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 340f74ab36b3e883ad7182d7bbc26926 2024-11-14T14:52:03,886 INFO [StoreOpener-340f74ab36b3e883ad7182d7bbc26926-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 340f74ab36b3e883ad7182d7bbc26926 2024-11-14T14:52:03,887 INFO [StoreOpener-340f74ab36b3e883ad7182d7bbc26926-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 340f74ab36b3e883ad7182d7bbc26926 columnFamilyName a 2024-11-14T14:52:03,887 DEBUG [StoreOpener-340f74ab36b3e883ad7182d7bbc26926-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T14:52:03,888 INFO [StoreOpener-340f74ab36b3e883ad7182d7bbc26926-1 {}] regionserver.HStore(327): Store=340f74ab36b3e883ad7182d7bbc26926/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T14:52:03,888 INFO [StoreOpener-340f74ab36b3e883ad7182d7bbc26926-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 340f74ab36b3e883ad7182d7bbc26926 2024-11-14T14:52:03,890 INFO [StoreOpener-340f74ab36b3e883ad7182d7bbc26926-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 340f74ab36b3e883ad7182d7bbc26926 columnFamilyName b 2024-11-14T14:52:03,890 DEBUG [StoreOpener-340f74ab36b3e883ad7182d7bbc26926-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T14:52:03,890 INFO [StoreOpener-340f74ab36b3e883ad7182d7bbc26926-1 {}] regionserver.HStore(327): Store=340f74ab36b3e883ad7182d7bbc26926/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T14:52:03,890 INFO [StoreOpener-340f74ab36b3e883ad7182d7bbc26926-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 340f74ab36b3e883ad7182d7bbc26926 2024-11-14T14:52:03,892 INFO [StoreOpener-340f74ab36b3e883ad7182d7bbc26926-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 340f74ab36b3e883ad7182d7bbc26926 columnFamilyName c 2024-11-14T14:52:03,892 DEBUG [StoreOpener-340f74ab36b3e883ad7182d7bbc26926-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T14:52:03,892 INFO [StoreOpener-340f74ab36b3e883ad7182d7bbc26926-1 {}] regionserver.HStore(327): Store=340f74ab36b3e883ad7182d7bbc26926/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T14:52:03,893 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 340f74ab36b3e883ad7182d7bbc26926 2024-11-14T14:52:03,893 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34445/hbase/data/default/test2727/340f74ab36b3e883ad7182d7bbc26926 2024-11-14T14:52:03,896 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 2 recovered edits file(s) under hdfs://localhost:34445/hbase/data/default/test2727/340f74ab36b3e883ad7182d7bbc26926 2024-11-14T14:52:03,897 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:34445/hbase/data/default/test2727/340f74ab36b3e883ad7182d7bbc26926/recovered.edits/0000000000000003000 2024-11-14T14:52:03,925 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-14T14:52:03,962 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 3000, skipped 0, firstSequenceIdInLog=1, maxSequenceIdInLog=3000, path=hdfs://localhost:34445/hbase/data/default/test2727/340f74ab36b3e883ad7182d7bbc26926/recovered.edits/0000000000000003000 2024-11-14T14:52:03,964 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:34445/hbase/data/default/test2727/340f74ab36b3e883ad7182d7bbc26926/recovered.edits/0000000000000006000 2024-11-14T14:52:04,008 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-14T14:52:04,034 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 3000, skipped 0, firstSequenceIdInLog=3001, maxSequenceIdInLog=6000, path=hdfs://localhost:34445/hbase/data/default/test2727/340f74ab36b3e883ad7182d7bbc26926/recovered.edits/0000000000000006000 2024-11-14T14:52:04,034 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 340f74ab36b3e883ad7182d7bbc26926 3/3 column families, dataSize=215.51 KB heapSize=657 KB 2024-11-14T14:52:04,054 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/hbase/data/default/test2727/340f74ab36b3e883ad7182d7bbc26926/.tmp/a/5147024a072d47748e10a1ced038c9fa is 41, key is test2727/a:100/1731595923472/Put/seqid=0 2024-11-14T14:52:04,067 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42819 is added to blk_1073741871_1048 (size=84227) 2024-11-14T14:52:04,068 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44549 is added to blk_1073741871_1048 (size=84227) 2024-11-14T14:52:04,068 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36189 is added to blk_1073741871_1048 (size=84227) 2024-11-14T14:52:04,467 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=71.84 KB at sequenceid=6000 (bloomFilter=true), to=hdfs://localhost:34445/hbase/data/default/test2727/340f74ab36b3e883ad7182d7bbc26926/.tmp/a/5147024a072d47748e10a1ced038c9fa 2024-11-14T14:52:04,495 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/hbase/data/default/test2727/340f74ab36b3e883ad7182d7bbc26926/.tmp/b/2e497f97ea614defaf9576f5d37a7ca8 is 41, key is test2727/b:100/1731595923522/Put/seqid=0 2024-11-14T14:52:04,500 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44549 is added to blk_1073741872_1049 (size=84609) 2024-11-14T14:52:04,500 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42819 is added to blk_1073741872_1049 (size=84609) 2024-11-14T14:52:04,501 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36189 is added to blk_1073741872_1049 (size=84609) 2024-11-14T14:52:04,501 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=71.84 KB at sequenceid=6000 (bloomFilter=true), to=hdfs://localhost:34445/hbase/data/default/test2727/340f74ab36b3e883ad7182d7bbc26926/.tmp/b/2e497f97ea614defaf9576f5d37a7ca8 2024-11-14T14:52:04,527 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/hbase/data/default/test2727/340f74ab36b3e883ad7182d7bbc26926/.tmp/c/8a9df39390c345fbbb27a73f75d29526 is 41, key is test2727/c:100/1731595923594/Put/seqid=0 2024-11-14T14:52:04,534 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36189 is added to blk_1073741873_1050 (size=84609) 2024-11-14T14:52:04,534 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44549 is added to blk_1073741873_1050 (size=84609) 2024-11-14T14:52:04,535 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42819 is added to blk_1073741873_1050 (size=84609) 2024-11-14T14:52:04,536 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=71.84 KB at sequenceid=6000 (bloomFilter=true), to=hdfs://localhost:34445/hbase/data/default/test2727/340f74ab36b3e883ad7182d7bbc26926/.tmp/c/8a9df39390c345fbbb27a73f75d29526 2024-11-14T14:52:04,543 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/hbase/data/default/test2727/340f74ab36b3e883ad7182d7bbc26926/.tmp/a/5147024a072d47748e10a1ced038c9fa as hdfs://localhost:34445/hbase/data/default/test2727/340f74ab36b3e883ad7182d7bbc26926/a/5147024a072d47748e10a1ced038c9fa 2024-11-14T14:52:04,551 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/hbase/data/default/test2727/340f74ab36b3e883ad7182d7bbc26926/a/5147024a072d47748e10a1ced038c9fa, entries=2000, sequenceid=6000, filesize=82.3 K 2024-11-14T14:52:04,552 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/hbase/data/default/test2727/340f74ab36b3e883ad7182d7bbc26926/.tmp/b/2e497f97ea614defaf9576f5d37a7ca8 as hdfs://localhost:34445/hbase/data/default/test2727/340f74ab36b3e883ad7182d7bbc26926/b/2e497f97ea614defaf9576f5d37a7ca8 2024-11-14T14:52:04,560 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/hbase/data/default/test2727/340f74ab36b3e883ad7182d7bbc26926/b/2e497f97ea614defaf9576f5d37a7ca8, entries=2000, sequenceid=6000, filesize=82.6 K 2024-11-14T14:52:04,561 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/hbase/data/default/test2727/340f74ab36b3e883ad7182d7bbc26926/.tmp/c/8a9df39390c345fbbb27a73f75d29526 as hdfs://localhost:34445/hbase/data/default/test2727/340f74ab36b3e883ad7182d7bbc26926/c/8a9df39390c345fbbb27a73f75d29526 2024-11-14T14:52:04,570 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/hbase/data/default/test2727/340f74ab36b3e883ad7182d7bbc26926/c/8a9df39390c345fbbb27a73f75d29526, entries=2000, sequenceid=6000, filesize=82.6 K 2024-11-14T14:52:04,570 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~215.51 KB/220680, heapSize ~656.95 KB/672720, currentSize=0 B/0 for 340f74ab36b3e883ad7182d7bbc26926 in 536ms, sequenceid=6000, compaction requested=false; wal=null 2024-11-14T14:52:04,571 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:34445/hbase/data/default/test2727/340f74ab36b3e883ad7182d7bbc26926/recovered.edits/0000000000000003000 2024-11-14T14:52:04,572 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:34445/hbase/data/default/test2727/340f74ab36b3e883ad7182d7bbc26926/recovered.edits/0000000000000006000 2024-11-14T14:52:04,573 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 340f74ab36b3e883ad7182d7bbc26926 2024-11-14T14:52:04,573 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 340f74ab36b3e883ad7182d7bbc26926 2024-11-14T14:52:04,574 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table test2727 descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-14T14:52:04,577 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 340f74ab36b3e883ad7182d7bbc26926 2024-11-14T14:52:04,580 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34445/hbase/data/default/test2727/340f74ab36b3e883ad7182d7bbc26926/recovered.edits/6000.seqid, newMaxSeqId=6000, maxSeqId=1 2024-11-14T14:52:04,581 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 340f74ab36b3e883ad7182d7bbc26926; next sequenceid=6001; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70292233, jitterRate=0.04743589460849762}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-14T14:52:04,583 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 340f74ab36b3e883ad7182d7bbc26926: Writing region info on filesystem at 1731595923884Initializing all the Stores at 1731595923885 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731595923885Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731595923885Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731595923886 (+1 ms)Obtaining lock to block concurrent updates at 1731595924034 (+148 ms)Preparing flush snapshotting stores in 340f74ab36b3e883ad7182d7bbc26926 at 1731595924034Finished memstore snapshotting test2727,,1731595922865.340f74ab36b3e883ad7182d7bbc26926., syncing WAL and waiting on mvcc, flushsize=dataSize=220680, getHeapSize=672720, getOffHeapSize=0, getCellsCount=6000 at 1731595924034Flushing stores of test2727,,1731595922865.340f74ab36b3e883ad7182d7bbc26926. at 1731595924034Flushing 340f74ab36b3e883ad7182d7bbc26926/a: creating writer at 1731595924034Flushing 340f74ab36b3e883ad7182d7bbc26926/a: appending metadata at 1731595924054 (+20 ms)Flushing 340f74ab36b3e883ad7182d7bbc26926/a: closing flushed file at 1731595924054Flushing 340f74ab36b3e883ad7182d7bbc26926/b: creating writer at 1731595924474 (+420 ms)Flushing 340f74ab36b3e883ad7182d7bbc26926/b: appending metadata at 1731595924493 (+19 ms)Flushing 340f74ab36b3e883ad7182d7bbc26926/b: closing flushed file at 1731595924493Flushing 340f74ab36b3e883ad7182d7bbc26926/c: creating writer at 1731595924507 (+14 ms)Flushing 340f74ab36b3e883ad7182d7bbc26926/c: appending metadata at 1731595924526 (+19 ms)Flushing 340f74ab36b3e883ad7182d7bbc26926/c: closing flushed file at 1731595924526Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@62cf9f2d: reopening flushed file at 1731595924542 (+16 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1c7363ad: reopening flushed file at 1731595924551 (+9 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5a31ee04: reopening flushed file at 1731595924560 (+9 ms)Finished flush of dataSize ~215.51 KB/220680, heapSize ~656.95 KB/672720, currentSize=0 B/0 for 340f74ab36b3e883ad7182d7bbc26926 in 536ms, sequenceid=6000, compaction requested=false; wal=null at 1731595924570 (+10 ms)Cleaning up temporary data from old regions at 1731595924573 (+3 ms)Region opened successfully at 1731595924583 (+10 ms) 2024-11-14T14:52:04,584 DEBUG [Time-limited test {}] wal.AbstractTestWALReplay(320): region.getOpenSeqNum(): 6001, wal3.id: 0 2024-11-14T14:52:04,585 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 340f74ab36b3e883ad7182d7bbc26926, disabling compactions & flushes 2024-11-14T14:52:04,585 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region test2727,,1731595922865.340f74ab36b3e883ad7182d7bbc26926. 2024-11-14T14:52:04,585 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on test2727,,1731595922865.340f74ab36b3e883ad7182d7bbc26926. 2024-11-14T14:52:04,585 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on test2727,,1731595922865.340f74ab36b3e883ad7182d7bbc26926. after waiting 0 ms 2024-11-14T14:52:04,585 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region test2727,,1731595922865.340f74ab36b3e883ad7182d7bbc26926. 2024-11-14T14:52:04,586 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed test2727,,1731595922865.340f74ab36b3e883ad7182d7bbc26926. 2024-11-14T14:52:04,587 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 340f74ab36b3e883ad7182d7bbc26926: Waiting for close lock at 1731595924584Disabling compacts and flushes for region at 1731595924584Disabling writes for close at 1731595924585 (+1 ms)Writing region close event to WAL at 1731595924586 (+1 ms)Closed at 1731595924586 2024-11-14T14:52:04,592 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44549 is added to blk_1073741870_1047 (size=175) 2024-11-14T14:52:04,592 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /hbase/WALs/test2727-manual,16010,1731595922863/wal.1731595923829 not finished, retry = 0 2024-11-14T14:52:04,593 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36189 is added to blk_1073741870_1047 (size=175) 2024-11-14T14:52:04,593 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42819 is added to blk_1073741870_1047 (size=175) 2024-11-14T14:52:04,696 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-14T14:52:04,696 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1731595923829) 2024-11-14T14:52:04,710 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#test2727 Thread=405 (was 403) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1791616411_22 at /127.0.0.1:57876 [Waiting for operation #9] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-14-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1791616411_22 at /127.0.0.1:53222 [Waiting for operation #5] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1791616411_22 at /127.0.0.1:33652 [Waiting for operation #9] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-14-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-14-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=927 (was 865) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=183 (was 183), ProcessCount=11 (was 11), AvailableMemoryMB=7627 (was 7835) 2024-11-14T14:52:04,723 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testSequentialEditLogSeqNum Thread=405, OpenFileDescriptor=927, MaxFileDescriptor=1048576, SystemLoadAverage=183, ProcessCount=11, AvailableMemoryMB=7626 2024-11-14T14:52:04,738 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-14T14:52:04,745 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:34445/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1731595924738, archiveDir=hdfs://localhost:34445/hbase/oldWALs, maxLogs=32 2024-11-14T14:52:04,746 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731595924746 2024-11-14T14:52:04,780 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testsequentialeditlogseqnum-manual,16010,1731595924738/wal.1731595924746 2024-11-14T14:52:04,782 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new MockWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33623:33623),(127.0.0.1/127.0.0.1:33515:33515),(127.0.0.1/127.0.0.1:34777:34777)] 2024-11-14T14:52:04,783 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 97b366da73ced1976c6a8ed40053f139, NAME => 'testSequentialEditLogSeqNum,,1731595924739.97b366da73ced1976c6a8ed40053f139.', STARTKEY => '', ENDKEY => ''} 2024-11-14T14:52:04,784 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testSequentialEditLogSeqNum,,1731595924739.97b366da73ced1976c6a8ed40053f139.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T14:52:04,784 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 97b366da73ced1976c6a8ed40053f139 2024-11-14T14:52:04,784 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 97b366da73ced1976c6a8ed40053f139 2024-11-14T14:52:04,785 WARN [Time-limited test {}] regionserver.HRegionFileSystem(836): hdfs://localhost:34445/hbase/data/default/testSequentialEditLogSeqNum/97b366da73ced1976c6a8ed40053f139 doesn't exist for region: 97b366da73ced1976c6a8ed40053f139 on table testSequentialEditLogSeqNum 2024-11-14T14:52:04,785 WARN [Time-limited test {}] regionserver.HRegionFileSystem(854): .regioninfo file not found for region: 97b366da73ced1976c6a8ed40053f139 on table testSequentialEditLogSeqNum 2024-11-14T14:52:04,793 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44549 is added to blk_1073741875_1052 (size=62) 2024-11-14T14:52:04,793 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36189 is added to blk_1073741875_1052 (size=62) 2024-11-14T14:52:04,793 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42819 is added to blk_1073741875_1052 (size=62) 2024-11-14T14:52:04,795 INFO [StoreOpener-97b366da73ced1976c6a8ed40053f139-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 97b366da73ced1976c6a8ed40053f139 2024-11-14T14:52:04,797 INFO [StoreOpener-97b366da73ced1976c6a8ed40053f139-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 97b366da73ced1976c6a8ed40053f139 columnFamilyName a 2024-11-14T14:52:04,797 DEBUG [StoreOpener-97b366da73ced1976c6a8ed40053f139-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T14:52:04,798 INFO [StoreOpener-97b366da73ced1976c6a8ed40053f139-1 {}] regionserver.HStore(327): Store=97b366da73ced1976c6a8ed40053f139/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T14:52:04,798 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 97b366da73ced1976c6a8ed40053f139 2024-11-14T14:52:04,798 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34445/hbase/data/default/testSequentialEditLogSeqNum/97b366da73ced1976c6a8ed40053f139 2024-11-14T14:52:04,799 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34445/hbase/data/default/testSequentialEditLogSeqNum/97b366da73ced1976c6a8ed40053f139 2024-11-14T14:52:04,799 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 97b366da73ced1976c6a8ed40053f139 2024-11-14T14:52:04,799 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 97b366da73ced1976c6a8ed40053f139 2024-11-14T14:52:04,801 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 97b366da73ced1976c6a8ed40053f139 2024-11-14T14:52:04,804 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34445/hbase/data/default/testSequentialEditLogSeqNum/97b366da73ced1976c6a8ed40053f139/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-14T14:52:04,805 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 97b366da73ced1976c6a8ed40053f139; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59933605, jitterRate=-0.10691969096660614}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-14T14:52:04,806 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 97b366da73ced1976c6a8ed40053f139: Writing region info on filesystem at 1731595924784Initializing all the Stores at 1731595924795 (+11 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731595924795Cleaning up temporary data from old regions at 1731595924799 (+4 ms)Region opened successfully at 1731595924805 (+6 ms) 2024-11-14T14:52:04,817 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 97b366da73ced1976c6a8ed40053f139 1/1 column families, dataSize=770 B heapSize=1.73 KB 2024-11-14T14:52:04,838 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/hbase/data/default/testSequentialEditLogSeqNum/97b366da73ced1976c6a8ed40053f139/.tmp/a/c0e056bb41134027b2cda02ce41758e6 is 81, key is testSequentialEditLogSeqNum/a:x0/1731595924806/Put/seqid=0 2024-11-14T14:52:04,845 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44549 is added to blk_1073741876_1053 (size=5833) 2024-11-14T14:52:04,845 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42819 is added to blk_1073741876_1053 (size=5833) 2024-11-14T14:52:04,846 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36189 is added to blk_1073741876_1053 (size=5833) 2024-11-14T14:52:04,846 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=770 B at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:34445/hbase/data/default/testSequentialEditLogSeqNum/97b366da73ced1976c6a8ed40053f139/.tmp/a/c0e056bb41134027b2cda02ce41758e6 2024-11-14T14:52:04,853 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/hbase/data/default/testSequentialEditLogSeqNum/97b366da73ced1976c6a8ed40053f139/.tmp/a/c0e056bb41134027b2cda02ce41758e6 as hdfs://localhost:34445/hbase/data/default/testSequentialEditLogSeqNum/97b366da73ced1976c6a8ed40053f139/a/c0e056bb41134027b2cda02ce41758e6 2024-11-14T14:52:04,860 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/hbase/data/default/testSequentialEditLogSeqNum/97b366da73ced1976c6a8ed40053f139/a/c0e056bb41134027b2cda02ce41758e6, entries=10, sequenceid=13, filesize=5.7 K 2024-11-14T14:52:04,861 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~770 B/770, heapSize ~1.72 KB/1760, currentSize=0 B/0 for 97b366da73ced1976c6a8ed40053f139 in 44ms, sequenceid=13, compaction requested=false 2024-11-14T14:52:04,862 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 97b366da73ced1976c6a8ed40053f139: 2024-11-14T14:52:04,867 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:52:04,867 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:52:04,868 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:52:04,868 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:52:04,868 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:52:04,870 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42819 is added to blk_1073741874_1051 (size=3256) 2024-11-14T14:52:04,870 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44549 is added to blk_1073741874_1051 (size=3256) 2024-11-14T14:52:04,870 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36189 is added to blk_1073741874_1051 (size=3256) 2024-11-14T14:52:04,887 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:34445/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1731595924738/wal.1731595924746, size=3.2 K (3256bytes) 2024-11-14T14:52:04,887 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:34445/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1731595924738/wal.1731595924746 2024-11-14T14:52:04,887 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:34445/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1731595924738/wal.1731595924746 after 0ms 2024-11-14T14:52:04,914 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-14T14:52:04,914 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:34445/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1731595924738/wal.1731595924746 took 27ms 2024-11-14T14:52:04,917 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:34445/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1731595924738/wal.1731595924746 so closing down 2024-11-14T14:52:04,917 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-14T14:52:04,918 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1731595924746.temp 2024-11-14T14:52:04,944 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:34445/hbase/data/default/testSequentialEditLogSeqNum/97b366da73ced1976c6a8ed40053f139/recovered.edits/0000000000000000003-wal.1731595924746.temp 2024-11-14T14:52:04,944 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-14T14:52:04,951 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36189 is added to blk_1073741877_1054 (size=2650) 2024-11-14T14:52:04,951 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44549 is added to blk_1073741877_1054 (size=2650) 2024-11-14T14:52:04,952 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42819 is added to blk_1073741877_1054 (size=2650) 2024-11-14T14:52:04,952 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:34445/hbase/data/default/testSequentialEditLogSeqNum/97b366da73ced1976c6a8ed40053f139/recovered.edits/0000000000000000003-wal.1731595924746.temp (wrote 15 edits, skipped 0 edits in 0 ms) 2024-11-14T14:52:04,954 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:34445/hbase/data/default/testSequentialEditLogSeqNum/97b366da73ced1976c6a8ed40053f139/recovered.edits/0000000000000000003-wal.1731595924746.temp to hdfs://localhost:34445/hbase/data/default/testSequentialEditLogSeqNum/97b366da73ced1976c6a8ed40053f139/recovered.edits/0000000000000000020 2024-11-14T14:52:04,954 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 17 edits across 1 Regions in 39 ms; skipped=2; WAL=hdfs://localhost:34445/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1731595924738/wal.1731595924746, size=3.2 K, length=3256, corrupted=false, cancelled=false 2024-11-14T14:52:04,954 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:34445/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1731595924738/wal.1731595924746, journal: Splitting hdfs://localhost:34445/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1731595924738/wal.1731595924746, size=3.2 K (3256bytes) at 1731595924887Finishing writing output for hdfs://localhost:34445/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1731595924738/wal.1731595924746 so closing down at 1731595924917 (+30 ms)Creating recovered edits writer path=hdfs://localhost:34445/hbase/data/default/testSequentialEditLogSeqNum/97b366da73ced1976c6a8ed40053f139/recovered.edits/0000000000000000003-wal.1731595924746.temp at 1731595924944 (+27 ms)3 split writer threads finished at 1731595924944Closed recovered edits writer path=hdfs://localhost:34445/hbase/data/default/testSequentialEditLogSeqNum/97b366da73ced1976c6a8ed40053f139/recovered.edits/0000000000000000003-wal.1731595924746.temp (wrote 15 edits, skipped 0 edits in 0 ms) at 1731595924952 (+8 ms)Rename recovered edits hdfs://localhost:34445/hbase/data/default/testSequentialEditLogSeqNum/97b366da73ced1976c6a8ed40053f139/recovered.edits/0000000000000000003-wal.1731595924746.temp to hdfs://localhost:34445/hbase/data/default/testSequentialEditLogSeqNum/97b366da73ced1976c6a8ed40053f139/recovered.edits/0000000000000000020 at 1731595924954 (+2 ms)Processed 17 edits across 1 Regions in 39 ms; skipped=2; WAL=hdfs://localhost:34445/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1731595924738/wal.1731595924746, size=3.2 K, length=3256, corrupted=false, cancelled=false at 1731595924954 2024-11-14T14:52:04,969 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testSequentialEditLogSeqNum Thread=410 (was 405) Potentially hanging thread: sync.4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1791616411_22 at /127.0.0.1:53222 [Waiting for operation #9] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: sync.3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1791616411_22 at /127.0.0.1:33652 [Waiting for operation #11] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: sync.1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) - Thread LEAK? -, OpenFileDescriptor=963 (was 927) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=183 (was 183), ProcessCount=11 (was 11), AvailableMemoryMB=7624 (was 7626) 2024-11-14T14:52:04,981 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testRegionMadeOfBulkLoadedFilesOnly Thread=410, OpenFileDescriptor=963, MaxFileDescriptor=1048576, SystemLoadAverage=183, ProcessCount=11, AvailableMemoryMB=7623 2024-11-14T14:52:04,997 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-14T14:52:04,999 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-14T14:52:05,016 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-14T14:52:05,019 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-19868467, suffix=, logDir=hdfs://localhost:34445/hbase/WALs/hregion-19868467, archiveDir=hdfs://localhost:34445/hbase/oldWALs, maxLogs=32 2024-11-14T14:52:05,033 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-19868467/hregion-19868467.1731595925020, exclude list is [], retry=0 2024-11-14T14:52:05,036 DEBUG [AsyncFSWAL-17-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42819,DS-c1b8ccdc-5e2c-4b3d-bb1e-6941b40dfb7e,DISK] 2024-11-14T14:52:05,036 DEBUG [AsyncFSWAL-17-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44549,DS-913adb4a-7946-4f46-896c-56bcdc4fcfa8,DISK] 2024-11-14T14:52:05,037 DEBUG [AsyncFSWAL-17-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36189,DS-5c245456-9209-4937-bfad-43a9a1b09820,DISK] 2024-11-14T14:52:05,064 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-19868467/hregion-19868467.1731595925020 2024-11-14T14:52:05,065 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34777:34777),(127.0.0.1/127.0.0.1:33515:33515),(127.0.0.1/127.0.0.1:33623:33623)] 2024-11-14T14:52:05,065 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 8f020bfa4f06c61730fa2b46b0562790, NAME => 'testRegionMadeOfBulkLoadedFilesOnly,,1731595924997.8f020bfa4f06c61730fa2b46b0562790.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testRegionMadeOfBulkLoadedFilesOnly', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34445/hbase 2024-11-14T14:52:05,075 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44549 is added to blk_1073741879_1056 (size=70) 2024-11-14T14:52:05,075 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36189 is added to blk_1073741879_1056 (size=70) 2024-11-14T14:52:05,075 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42819 is added to blk_1073741879_1056 (size=70) 2024-11-14T14:52:05,076 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testRegionMadeOfBulkLoadedFilesOnly,,1731595924997.8f020bfa4f06c61730fa2b46b0562790.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T14:52:05,078 INFO [StoreOpener-8f020bfa4f06c61730fa2b46b0562790-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 8f020bfa4f06c61730fa2b46b0562790 2024-11-14T14:52:05,080 INFO [StoreOpener-8f020bfa4f06c61730fa2b46b0562790-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8f020bfa4f06c61730fa2b46b0562790 columnFamilyName a 2024-11-14T14:52:05,080 DEBUG [StoreOpener-8f020bfa4f06c61730fa2b46b0562790-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T14:52:05,080 INFO [StoreOpener-8f020bfa4f06c61730fa2b46b0562790-1 {}] regionserver.HStore(327): Store=8f020bfa4f06c61730fa2b46b0562790/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T14:52:05,081 INFO [StoreOpener-8f020bfa4f06c61730fa2b46b0562790-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 8f020bfa4f06c61730fa2b46b0562790 2024-11-14T14:52:05,082 INFO [StoreOpener-8f020bfa4f06c61730fa2b46b0562790-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8f020bfa4f06c61730fa2b46b0562790 columnFamilyName b 2024-11-14T14:52:05,082 DEBUG [StoreOpener-8f020bfa4f06c61730fa2b46b0562790-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T14:52:05,083 INFO [StoreOpener-8f020bfa4f06c61730fa2b46b0562790-1 {}] regionserver.HStore(327): Store=8f020bfa4f06c61730fa2b46b0562790/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T14:52:05,083 INFO [StoreOpener-8f020bfa4f06c61730fa2b46b0562790-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 8f020bfa4f06c61730fa2b46b0562790 2024-11-14T14:52:05,084 INFO [StoreOpener-8f020bfa4f06c61730fa2b46b0562790-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8f020bfa4f06c61730fa2b46b0562790 columnFamilyName c 2024-11-14T14:52:05,084 DEBUG [StoreOpener-8f020bfa4f06c61730fa2b46b0562790-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T14:52:05,085 INFO [StoreOpener-8f020bfa4f06c61730fa2b46b0562790-1 {}] regionserver.HStore(327): Store=8f020bfa4f06c61730fa2b46b0562790/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T14:52:05,085 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 8f020bfa4f06c61730fa2b46b0562790 2024-11-14T14:52:05,085 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34445/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/8f020bfa4f06c61730fa2b46b0562790 2024-11-14T14:52:05,086 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34445/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/8f020bfa4f06c61730fa2b46b0562790 2024-11-14T14:52:05,087 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 8f020bfa4f06c61730fa2b46b0562790 2024-11-14T14:52:05,087 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 8f020bfa4f06c61730fa2b46b0562790 2024-11-14T14:52:05,087 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRegionMadeOfBulkLoadedFilesOnly descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-14T14:52:05,089 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 8f020bfa4f06c61730fa2b46b0562790 2024-11-14T14:52:05,091 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34445/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/8f020bfa4f06c61730fa2b46b0562790/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-14T14:52:05,092 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 8f020bfa4f06c61730fa2b46b0562790; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66489618, jitterRate=-0.009227484464645386}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-14T14:52:05,093 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 8f020bfa4f06c61730fa2b46b0562790: Writing region info on filesystem at 1731595925076Initializing all the Stores at 1731595925077 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731595925078 (+1 ms)Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731595925078Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731595925078Cleaning up temporary data from old regions at 1731595925087 (+9 ms)Region opened successfully at 1731595925093 (+6 ms) 2024-11-14T14:52:05,093 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 8f020bfa4f06c61730fa2b46b0562790, disabling compactions & flushes 2024-11-14T14:52:05,093 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testRegionMadeOfBulkLoadedFilesOnly,,1731595924997.8f020bfa4f06c61730fa2b46b0562790. 2024-11-14T14:52:05,093 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testRegionMadeOfBulkLoadedFilesOnly,,1731595924997.8f020bfa4f06c61730fa2b46b0562790. 2024-11-14T14:52:05,093 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testRegionMadeOfBulkLoadedFilesOnly,,1731595924997.8f020bfa4f06c61730fa2b46b0562790. after waiting 0 ms 2024-11-14T14:52:05,093 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testRegionMadeOfBulkLoadedFilesOnly,,1731595924997.8f020bfa4f06c61730fa2b46b0562790. 2024-11-14T14:52:05,094 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testRegionMadeOfBulkLoadedFilesOnly,,1731595924997.8f020bfa4f06c61730fa2b46b0562790. 2024-11-14T14:52:05,094 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 8f020bfa4f06c61730fa2b46b0562790: Waiting for close lock at 1731595925093Disabling compacts and flushes for region at 1731595925093Disabling writes for close at 1731595925093Writing region close event to WAL at 1731595925094 (+1 ms)Closed at 1731595925094 2024-11-14T14:52:05,097 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42819 is added to blk_1073741878_1055 (size=175) 2024-11-14T14:52:05,097 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44549 is added to blk_1073741878_1055 (size=175) 2024-11-14T14:52:05,097 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36189 is added to blk_1073741878_1055 (size=175) 2024-11-14T14:52:05,100 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-14T14:52:05,100 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-19868467:(num 1731595925020) 2024-11-14T14:52:05,100 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-14T14:52:05,102 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:34445/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731595924996, archiveDir=hdfs://localhost:34445/hbase/oldWALs, maxLogs=32 2024-11-14T14:52:05,123 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731595924996/wal.1731595925103, exclude list is [], retry=0 2024-11-14T14:52:05,126 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44549,DS-913adb4a-7946-4f46-896c-56bcdc4fcfa8,DISK] 2024-11-14T14:52:05,127 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42819,DS-c1b8ccdc-5e2c-4b3d-bb1e-6941b40dfb7e,DISK] 2024-11-14T14:52:05,127 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36189,DS-5c245456-9209-4937-bfad-43a9a1b09820,DISK] 2024-11-14T14:52:05,165 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731595924996/wal.1731595925103 2024-11-14T14:52:05,165 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33515:33515),(127.0.0.1/127.0.0.1:34777:34777),(127.0.0.1/127.0.0.1:33623:33623)] 2024-11-14T14:52:05,166 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 8f020bfa4f06c61730fa2b46b0562790, NAME => 'testRegionMadeOfBulkLoadedFilesOnly,,1731595924997.8f020bfa4f06c61730fa2b46b0562790.', STARTKEY => '', ENDKEY => ''} 2024-11-14T14:52:05,166 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testRegionMadeOfBulkLoadedFilesOnly,,1731595924997.8f020bfa4f06c61730fa2b46b0562790.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T14:52:05,166 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 8f020bfa4f06c61730fa2b46b0562790 2024-11-14T14:52:05,166 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 8f020bfa4f06c61730fa2b46b0562790 2024-11-14T14:52:05,167 INFO [StoreOpener-8f020bfa4f06c61730fa2b46b0562790-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 8f020bfa4f06c61730fa2b46b0562790 2024-11-14T14:52:05,169 INFO [StoreOpener-8f020bfa4f06c61730fa2b46b0562790-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8f020bfa4f06c61730fa2b46b0562790 columnFamilyName a 2024-11-14T14:52:05,169 DEBUG [StoreOpener-8f020bfa4f06c61730fa2b46b0562790-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T14:52:05,169 INFO [StoreOpener-8f020bfa4f06c61730fa2b46b0562790-1 {}] regionserver.HStore(327): Store=8f020bfa4f06c61730fa2b46b0562790/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T14:52:05,169 INFO [StoreOpener-8f020bfa4f06c61730fa2b46b0562790-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 8f020bfa4f06c61730fa2b46b0562790 2024-11-14T14:52:05,170 INFO [StoreOpener-8f020bfa4f06c61730fa2b46b0562790-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8f020bfa4f06c61730fa2b46b0562790 columnFamilyName b 2024-11-14T14:52:05,170 DEBUG [StoreOpener-8f020bfa4f06c61730fa2b46b0562790-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T14:52:05,171 INFO [StoreOpener-8f020bfa4f06c61730fa2b46b0562790-1 {}] regionserver.HStore(327): Store=8f020bfa4f06c61730fa2b46b0562790/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T14:52:05,171 INFO [StoreOpener-8f020bfa4f06c61730fa2b46b0562790-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 8f020bfa4f06c61730fa2b46b0562790 2024-11-14T14:52:05,171 INFO [StoreOpener-8f020bfa4f06c61730fa2b46b0562790-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8f020bfa4f06c61730fa2b46b0562790 columnFamilyName c 2024-11-14T14:52:05,171 DEBUG [StoreOpener-8f020bfa4f06c61730fa2b46b0562790-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T14:52:05,172 INFO [StoreOpener-8f020bfa4f06c61730fa2b46b0562790-1 {}] regionserver.HStore(327): Store=8f020bfa4f06c61730fa2b46b0562790/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T14:52:05,172 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 8f020bfa4f06c61730fa2b46b0562790 2024-11-14T14:52:05,173 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34445/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/8f020bfa4f06c61730fa2b46b0562790 2024-11-14T14:52:05,174 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34445/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/8f020bfa4f06c61730fa2b46b0562790 2024-11-14T14:52:05,175 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 8f020bfa4f06c61730fa2b46b0562790 2024-11-14T14:52:05,175 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 8f020bfa4f06c61730fa2b46b0562790 2024-11-14T14:52:05,175 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRegionMadeOfBulkLoadedFilesOnly descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-14T14:52:05,177 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 8f020bfa4f06c61730fa2b46b0562790 2024-11-14T14:52:05,177 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 8f020bfa4f06c61730fa2b46b0562790; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59517766, jitterRate=-0.11311617493629456}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-14T14:52:05,178 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 8f020bfa4f06c61730fa2b46b0562790: Writing region info on filesystem at 1731595925166Initializing all the Stores at 1731595925167 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731595925167Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731595925167Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731595925167Cleaning up temporary data from old regions at 1731595925175 (+8 ms)Region opened successfully at 1731595925178 (+3 ms) 2024-11-14T14:52:05,183 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile is 28, key is \x0D/a:a/1731595925182/Put/seqid=0 2024-11-14T14:52:05,189 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44549 is added to blk_1073741881_1058 (size=4826) 2024-11-14T14:52:05,190 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42819 is added to blk_1073741881_1058 (size=4826) 2024-11-14T14:52:05,190 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36189 is added to blk_1073741881_1058 (size=4826) 2024-11-14T14:52:05,191 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:34445/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile for inclusion in 8f020bfa4f06c61730fa2b46b0562790/a 2024-11-14T14:52:05,199 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first= last=z 2024-11-14T14:52:05,199 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-11-14T14:52:05,199 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 8f020bfa4f06c61730fa2b46b0562790: 2024-11-14T14:52:05,201 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile as hdfs://localhost:34445/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/8f020bfa4f06c61730fa2b46b0562790/a/10d989552a4d4805b3f67516b0f4888a_SeqId_3_ 2024-11-14T14:52:05,202 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:34445/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile into 8f020bfa4f06c61730fa2b46b0562790/a as hdfs://localhost:34445/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/8f020bfa4f06c61730fa2b46b0562790/a/10d989552a4d4805b3f67516b0f4888a_SeqId_3_ - updating store file list. 2024-11-14T14:52:05,208 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for 10d989552a4d4805b3f67516b0f4888a_SeqId_3_: NONE, but ROW specified in column family configuration 2024-11-14T14:52:05,209 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:34445/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/8f020bfa4f06c61730fa2b46b0562790/a/10d989552a4d4805b3f67516b0f4888a_SeqId_3_ into 8f020bfa4f06c61730fa2b46b0562790/a 2024-11-14T14:52:05,209 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:34445/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile into 8f020bfa4f06c61730fa2b46b0562790/a (new location: hdfs://localhost:34445/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/8f020bfa4f06c61730fa2b46b0562790/a/10d989552a4d4805b3f67516b0f4888a_SeqId_3_) 2024-11-14T14:52:05,251 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:34445/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731595924996/wal.1731595925103, size=0 (0bytes) 2024-11-14T14:52:05,252 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:34445/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731595924996/wal.1731595925103 might be still open, length is 0 2024-11-14T14:52:05,252 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:34445/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731595924996/wal.1731595925103 2024-11-14T14:52:05,252 WARN [IPC Server handler 4 on default port 34445 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731595924996/wal.1731595925103 has not been closed. Lease recovery is in progress. RecoveryId = 1059 for block blk_1073741880_1057 2024-11-14T14:52:05,252 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:34445/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731595924996/wal.1731595925103 after 0ms 2024-11-14T14:52:06,689 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1791616411_22 at /127.0.0.1:57960 [Receiving block BP-1055376130-172.17.0.2-1731595909704:blk_1073741880_1057] {}] datanode.DataXceiver(331): 127.0.0.1:44549:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:57960 dst: /127.0.0.1:44549 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:44549 remote=/127.0.0.1:57960]. Total timeout mills is 60000, 58527 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:52:06,690 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1791616411_22 at /127.0.0.1:33854 [Receiving block BP-1055376130-172.17.0.2-1731595909704:blk_1073741880_1057] {}] datanode.DataXceiver(331): 127.0.0.1:42819:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:33854 dst: /127.0.0.1:42819 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:52:06,690 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1791616411_22 at /127.0.0.1:53314 [Receiving block BP-1055376130-172.17.0.2-1731595909704:blk_1073741880_1057] {}] datanode.DataXceiver(331): 127.0.0.1:36189:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:53314 dst: /127.0.0.1:36189 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:52:06,693 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44549 is added to blk_1073741880_1059 (size=606) 2024-11-14T14:52:06,693 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42819 is added to blk_1073741880_1059 (size=606) 2024-11-14T14:52:09,253 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:34445/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731595924996/wal.1731595925103 after 4001ms 2024-11-14T14:52:09,282 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-14T14:52:09,283 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:34445/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731595924996/wal.1731595925103 took 4032ms 2024-11-14T14:52:09,285 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:34445/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731595924996/wal.1731595925103; continuing. 2024-11-14T14:52:09,285 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:34445/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731595924996/wal.1731595925103 so closing down 2024-11-14T14:52:09,285 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-14T14:52:09,287 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000005-wal.1731595925103.temp 2024-11-14T14:52:09,312 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:34445/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/8f020bfa4f06c61730fa2b46b0562790/recovered.edits/0000000000000000005-wal.1731595925103.temp 2024-11-14T14:52:09,312 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-14T14:52:09,320 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42819 is added to blk_1073741882_1060 (size=329) 2024-11-14T14:52:09,320 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44549 is added to blk_1073741882_1060 (size=329) 2024-11-14T14:52:09,320 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36189 is added to blk_1073741882_1060 (size=329) 2024-11-14T14:52:09,320 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:34445/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/8f020bfa4f06c61730fa2b46b0562790/recovered.edits/0000000000000000005-wal.1731595925103.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-11-14T14:52:09,322 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:34445/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/8f020bfa4f06c61730fa2b46b0562790/recovered.edits/0000000000000000005-wal.1731595925103.temp to hdfs://localhost:34445/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/8f020bfa4f06c61730fa2b46b0562790/recovered.edits/0000000000000000005 2024-11-14T14:52:09,322 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 2 edits across 1 Regions in 39 ms; skipped=1; WAL=hdfs://localhost:34445/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731595924996/wal.1731595925103, size=0, length=0, corrupted=false, cancelled=false 2024-11-14T14:52:09,322 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:34445/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731595924996/wal.1731595925103, journal: Splitting hdfs://localhost:34445/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731595924996/wal.1731595925103, size=0 (0bytes) at 1731595925252Finishing writing output for hdfs://localhost:34445/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731595924996/wal.1731595925103 so closing down at 1731595929285 (+4033 ms)Creating recovered edits writer path=hdfs://localhost:34445/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/8f020bfa4f06c61730fa2b46b0562790/recovered.edits/0000000000000000005-wal.1731595925103.temp at 1731595929312 (+27 ms)3 split writer threads finished at 1731595929312Closed recovered edits writer path=hdfs://localhost:34445/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/8f020bfa4f06c61730fa2b46b0562790/recovered.edits/0000000000000000005-wal.1731595925103.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1731595929321 (+9 ms)Rename recovered edits hdfs://localhost:34445/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/8f020bfa4f06c61730fa2b46b0562790/recovered.edits/0000000000000000005-wal.1731595925103.temp to hdfs://localhost:34445/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/8f020bfa4f06c61730fa2b46b0562790/recovered.edits/0000000000000000005 at 1731595929322 (+1 ms)Processed 2 edits across 1 Regions in 39 ms; skipped=1; WAL=hdfs://localhost:34445/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731595924996/wal.1731595925103, size=0, length=0, corrupted=false, cancelled=false at 1731595929322 2024-11-14T14:52:09,324 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:34445/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731595924996/wal.1731595925103 to hdfs://localhost:34445/hbase/oldWALs/wal.1731595925103 2024-11-14T14:52:09,325 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:34445/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/8f020bfa4f06c61730fa2b46b0562790/recovered.edits/0000000000000000005 2024-11-14T14:52:09,325 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-14T14:52:09,327 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:34445/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731595924996, archiveDir=hdfs://localhost:34445/hbase/oldWALs, maxLogs=32 2024-11-14T14:52:09,339 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731595924996/wal.1731595929327, exclude list is [], retry=0 2024-11-14T14:52:09,342 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44549,DS-913adb4a-7946-4f46-896c-56bcdc4fcfa8,DISK] 2024-11-14T14:52:09,342 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36189,DS-5c245456-9209-4937-bfad-43a9a1b09820,DISK] 2024-11-14T14:52:09,342 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42819,DS-c1b8ccdc-5e2c-4b3d-bb1e-6941b40dfb7e,DISK] 2024-11-14T14:52:09,369 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731595924996/wal.1731595929327 2024-11-14T14:52:09,369 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33515:33515),(127.0.0.1/127.0.0.1:33623:33623),(127.0.0.1/127.0.0.1:34777:34777)] 2024-11-14T14:52:09,369 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 8f020bfa4f06c61730fa2b46b0562790, NAME => 'testRegionMadeOfBulkLoadedFilesOnly,,1731595924997.8f020bfa4f06c61730fa2b46b0562790.', STARTKEY => '', ENDKEY => ''} 2024-11-14T14:52:09,370 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testRegionMadeOfBulkLoadedFilesOnly,,1731595924997.8f020bfa4f06c61730fa2b46b0562790.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T14:52:09,370 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 8f020bfa4f06c61730fa2b46b0562790 2024-11-14T14:52:09,370 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 8f020bfa4f06c61730fa2b46b0562790 2024-11-14T14:52:09,372 INFO [StoreOpener-8f020bfa4f06c61730fa2b46b0562790-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 8f020bfa4f06c61730fa2b46b0562790 2024-11-14T14:52:09,373 INFO [StoreOpener-8f020bfa4f06c61730fa2b46b0562790-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8f020bfa4f06c61730fa2b46b0562790 columnFamilyName a 2024-11-14T14:52:09,373 DEBUG [StoreOpener-8f020bfa4f06c61730fa2b46b0562790-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T14:52:09,379 DEBUG [StoreFileOpener-8f020bfa4f06c61730fa2b46b0562790-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for 10d989552a4d4805b3f67516b0f4888a_SeqId_3_: NONE, but ROW specified in column family configuration 2024-11-14T14:52:09,379 DEBUG [StoreOpener-8f020bfa4f06c61730fa2b46b0562790-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:34445/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/8f020bfa4f06c61730fa2b46b0562790/a/10d989552a4d4805b3f67516b0f4888a_SeqId_3_ 2024-11-14T14:52:09,379 INFO [StoreOpener-8f020bfa4f06c61730fa2b46b0562790-1 {}] regionserver.HStore(327): Store=8f020bfa4f06c61730fa2b46b0562790/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T14:52:09,380 INFO [StoreOpener-8f020bfa4f06c61730fa2b46b0562790-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 8f020bfa4f06c61730fa2b46b0562790 2024-11-14T14:52:09,380 INFO [StoreOpener-8f020bfa4f06c61730fa2b46b0562790-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8f020bfa4f06c61730fa2b46b0562790 columnFamilyName b 2024-11-14T14:52:09,381 DEBUG [StoreOpener-8f020bfa4f06c61730fa2b46b0562790-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T14:52:09,381 INFO [StoreOpener-8f020bfa4f06c61730fa2b46b0562790-1 {}] regionserver.HStore(327): Store=8f020bfa4f06c61730fa2b46b0562790/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T14:52:09,381 INFO [StoreOpener-8f020bfa4f06c61730fa2b46b0562790-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 8f020bfa4f06c61730fa2b46b0562790 2024-11-14T14:52:09,382 INFO [StoreOpener-8f020bfa4f06c61730fa2b46b0562790-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8f020bfa4f06c61730fa2b46b0562790 columnFamilyName c 2024-11-14T14:52:09,382 DEBUG [StoreOpener-8f020bfa4f06c61730fa2b46b0562790-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T14:52:09,382 INFO [StoreOpener-8f020bfa4f06c61730fa2b46b0562790-1 {}] regionserver.HStore(327): Store=8f020bfa4f06c61730fa2b46b0562790/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T14:52:09,383 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 8f020bfa4f06c61730fa2b46b0562790 2024-11-14T14:52:09,383 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34445/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/8f020bfa4f06c61730fa2b46b0562790 2024-11-14T14:52:09,385 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:34445/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/8f020bfa4f06c61730fa2b46b0562790 2024-11-14T14:52:09,386 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:34445/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/8f020bfa4f06c61730fa2b46b0562790/recovered.edits/0000000000000000005 2024-11-14T14:52:09,413 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-14T14:52:09,414 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 1, skipped 0, firstSequenceIdInLog=5, maxSequenceIdInLog=5, path=hdfs://localhost:34445/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/8f020bfa4f06c61730fa2b46b0562790/recovered.edits/0000000000000000005 2024-11-14T14:52:09,414 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 8f020bfa4f06c61730fa2b46b0562790 3/3 column families, dataSize=58 B heapSize=904 B 2024-11-14T14:52:09,428 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/8f020bfa4f06c61730fa2b46b0562790/.tmp/a/553168de651546978d6bce72076cc010 is 62, key is testRegionMadeOfBulkLoadedFilesOnly/a:a/1731595925215/Put/seqid=0 2024-11-14T14:52:09,435 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44549 is added to blk_1073741884_1062 (size=5149) 2024-11-14T14:52:09,435 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42819 is added to blk_1073741884_1062 (size=5149) 2024-11-14T14:52:09,435 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36189 is added to blk_1073741884_1062 (size=5149) 2024-11-14T14:52:09,436 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58 B at sequenceid=5 (bloomFilter=true), to=hdfs://localhost:34445/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/8f020bfa4f06c61730fa2b46b0562790/.tmp/a/553168de651546978d6bce72076cc010 2024-11-14T14:52:09,441 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/8f020bfa4f06c61730fa2b46b0562790/.tmp/a/553168de651546978d6bce72076cc010 as hdfs://localhost:34445/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/8f020bfa4f06c61730fa2b46b0562790/a/553168de651546978d6bce72076cc010 2024-11-14T14:52:09,447 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/8f020bfa4f06c61730fa2b46b0562790/a/553168de651546978d6bce72076cc010, entries=1, sequenceid=5, filesize=5.0 K 2024-11-14T14:52:09,447 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~58 B/58, heapSize ~376 B/376, currentSize=0 B/0 for 8f020bfa4f06c61730fa2b46b0562790 in 33ms, sequenceid=5, compaction requested=false; wal=null 2024-11-14T14:52:09,448 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:34445/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/8f020bfa4f06c61730fa2b46b0562790/recovered.edits/0000000000000000005 2024-11-14T14:52:09,449 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 8f020bfa4f06c61730fa2b46b0562790 2024-11-14T14:52:09,449 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 8f020bfa4f06c61730fa2b46b0562790 2024-11-14T14:52:09,450 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRegionMadeOfBulkLoadedFilesOnly descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-14T14:52:09,452 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 8f020bfa4f06c61730fa2b46b0562790 2024-11-14T14:52:09,454 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34445/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/8f020bfa4f06c61730fa2b46b0562790/recovered.edits/5.seqid, newMaxSeqId=5, maxSeqId=1 2024-11-14T14:52:09,455 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 8f020bfa4f06c61730fa2b46b0562790; next sequenceid=6; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63547945, jitterRate=-0.0530618280172348}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-14T14:52:09,455 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 8f020bfa4f06c61730fa2b46b0562790: Writing region info on filesystem at 1731595929370Initializing all the Stores at 1731595929371 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731595929371Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731595929371Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731595929371Obtaining lock to block concurrent updates at 1731595929414 (+43 ms)Preparing flush snapshotting stores in 8f020bfa4f06c61730fa2b46b0562790 at 1731595929414Finished memstore snapshotting testRegionMadeOfBulkLoadedFilesOnly,,1731595924997.8f020bfa4f06c61730fa2b46b0562790., syncing WAL and waiting on mvcc, flushsize=dataSize=58, getHeapSize=856, getOffHeapSize=0, getCellsCount=1 at 1731595929414Flushing stores of testRegionMadeOfBulkLoadedFilesOnly,,1731595924997.8f020bfa4f06c61730fa2b46b0562790. at 1731595929414Flushing 8f020bfa4f06c61730fa2b46b0562790/a: creating writer at 1731595929414Flushing 8f020bfa4f06c61730fa2b46b0562790/a: appending metadata at 1731595929428 (+14 ms)Flushing 8f020bfa4f06c61730fa2b46b0562790/a: closing flushed file at 1731595929428Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3bd9f721: reopening flushed file at 1731595929440 (+12 ms)Finished flush of dataSize ~58 B/58, heapSize ~376 B/376, currentSize=0 B/0 for 8f020bfa4f06c61730fa2b46b0562790 in 33ms, sequenceid=5, compaction requested=false; wal=null at 1731595929447 (+7 ms)Cleaning up temporary data from old regions at 1731595929449 (+2 ms)Region opened successfully at 1731595929455 (+6 ms) 2024-11-14T14:52:09,460 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 8f020bfa4f06c61730fa2b46b0562790, disabling compactions & flushes 2024-11-14T14:52:09,460 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testRegionMadeOfBulkLoadedFilesOnly,,1731595924997.8f020bfa4f06c61730fa2b46b0562790. 2024-11-14T14:52:09,460 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testRegionMadeOfBulkLoadedFilesOnly,,1731595924997.8f020bfa4f06c61730fa2b46b0562790. 2024-11-14T14:52:09,460 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testRegionMadeOfBulkLoadedFilesOnly,,1731595924997.8f020bfa4f06c61730fa2b46b0562790. after waiting 0 ms 2024-11-14T14:52:09,460 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testRegionMadeOfBulkLoadedFilesOnly,,1731595924997.8f020bfa4f06c61730fa2b46b0562790. 2024-11-14T14:52:09,461 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testRegionMadeOfBulkLoadedFilesOnly,,1731595924997.8f020bfa4f06c61730fa2b46b0562790. 2024-11-14T14:52:09,461 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 8f020bfa4f06c61730fa2b46b0562790: Waiting for close lock at 1731595929459Disabling compacts and flushes for region at 1731595929459Disabling writes for close at 1731595929460 (+1 ms)Writing region close event to WAL at 1731595929461 (+1 ms)Closed at 1731595929461 2024-11-14T14:52:09,464 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36189 is added to blk_1073741883_1061 (size=175) 2024-11-14T14:52:09,464 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42819 is added to blk_1073741883_1061 (size=175) 2024-11-14T14:52:09,464 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44549 is added to blk_1073741883_1061 (size=175) 2024-11-14T14:52:09,466 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-14T14:52:09,466 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1731595929327) 2024-11-14T14:52:09,482 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testRegionMadeOfBulkLoadedFilesOnly Thread=414 (was 410) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1083158566_22 at /127.0.0.1:33870 [Waiting for operation #9] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-17-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-17-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1083158566_22 at /127.0.0.1:53360 [Waiting for operation #3] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1083158566_22 at /127.0.0.1:58010 [Waiting for operation #5] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkinstestRegionMadeOfBulkLoadedFilesOnly@localhost:34445 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1319272049) connection to localhost/127.0.0.1:34445 from jenkinstestRegionMadeOfBulkLoadedFilesOnly java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: AsyncFSWAL-17-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1023 (was 963) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=176 (was 183), ProcessCount=11 (was 11), AvailableMemoryMB=7614 (was 7623) 2024-11-14T14:52:09,494 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsAfterRegionMovedWithMultiCF Thread=414, OpenFileDescriptor=1023, MaxFileDescriptor=1048576, SystemLoadAverage=176, ProcessCount=11, AvailableMemoryMB=7613 2024-11-14T14:52:09,509 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-14T14:52:09,514 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-14T14:52:09,519 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.AsyncConnectionImpl(321): The fetched master address is da1c6afcd1f9,42151,1731595912734 2024-11-14T14:52:09,521 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@3b0c4e7c 2024-11-14T14:52:09,522 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-14T14:52:09,524 INFO [HMaster-EventLoopGroup-2-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53456, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-14T14:52:09,528 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42151 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testReplayEditsAfterRegionMovedWithMultiCF', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-14T14:52:09,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42151 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF 2024-11-14T14:52:09,536 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_PRE_OPERATION 2024-11-14T14:52:09,538 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42151 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testReplayEditsAfterRegionMovedWithMultiCF" procId is: 4 2024-11-14T14:52:09,538 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T14:52:09,540 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-14T14:52:09,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42151 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-14T14:52:09,549 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42819 is added to blk_1073741885_1063 (size=694) 2024-11-14T14:52:09,549 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36189 is added to blk_1073741885_1063 (size=694) 2024-11-14T14:52:09,550 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44549 is added to blk_1073741885_1063 (size=694) 2024-11-14T14:52:09,552 INFO [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => cf7ad4aad0d0e3c5a9f31c16500a8e87, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1731595929525.cf7ad4aad0d0e3c5a9f31c16500a8e87.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsAfterRegionMovedWithMultiCF', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34445/user/jenkins/test-data/35161b07-018a-0344-8c43-1519f3812567 2024-11-14T14:52:09,559 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42819 is added to blk_1073741886_1064 (size=77) 2024-11-14T14:52:09,559 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36189 is added to blk_1073741886_1064 (size=77) 2024-11-14T14:52:09,559 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44549 is added to blk_1073741886_1064 (size=77) 2024-11-14T14:52:09,560 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1731595929525.cf7ad4aad0d0e3c5a9f31c16500a8e87.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T14:52:09,560 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1722): Closing cf7ad4aad0d0e3c5a9f31c16500a8e87, disabling compactions & flushes 2024-11-14T14:52:09,560 INFO [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1731595929525.cf7ad4aad0d0e3c5a9f31c16500a8e87. 2024-11-14T14:52:09,560 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731595929525.cf7ad4aad0d0e3c5a9f31c16500a8e87. 2024-11-14T14:52:09,560 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731595929525.cf7ad4aad0d0e3c5a9f31c16500a8e87. after waiting 0 ms 2024-11-14T14:52:09,560 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1731595929525.cf7ad4aad0d0e3c5a9f31c16500a8e87. 2024-11-14T14:52:09,560 INFO [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1731595929525.cf7ad4aad0d0e3c5a9f31c16500a8e87. 2024-11-14T14:52:09,560 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1676): Region close journal for cf7ad4aad0d0e3c5a9f31c16500a8e87: Waiting for close lock at 1731595929560Disabling compacts and flushes for region at 1731595929560Disabling writes for close at 1731595929560Writing region close event to WAL at 1731595929560Closed at 1731595929560 2024-11-14T14:52:09,562 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_ADD_TO_META 2024-11-14T14:52:09,568 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testReplayEditsAfterRegionMovedWithMultiCF,,1731595929525.cf7ad4aad0d0e3c5a9f31c16500a8e87.","families":{"info":[{"qualifier":"regioninfo","vlen":76,"tag":[],"timestamp":"1731595929562"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731595929562"}]},"ts":"1731595929562"} 2024-11-14T14:52:09,571 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-14T14:52:09,573 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-14T14:52:09,575 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testReplayEditsAfterRegionMovedWithMultiCF","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731595929573"}]},"ts":"1731595929573"} 2024-11-14T14:52:09,579 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testReplayEditsAfterRegionMovedWithMultiCF, state=ENABLING in hbase:meta 2024-11-14T14:52:09,580 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {da1c6afcd1f9=0} racks are {/default-rack=0} 2024-11-14T14:52:09,581 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-14T14:52:09,581 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-14T14:52:09,581 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-11-14T14:52:09,581 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-14T14:52:09,581 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-14T14:52:09,581 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-11-14T14:52:09,581 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-14T14:52:09,581 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-14T14:52:09,581 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-11-14T14:52:09,581 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-14T14:52:09,583 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=cf7ad4aad0d0e3c5a9f31c16500a8e87, ASSIGN}] 2024-11-14T14:52:09,584 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=cf7ad4aad0d0e3c5a9f31c16500a8e87, ASSIGN 2024-11-14T14:52:09,586 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=cf7ad4aad0d0e3c5a9f31c16500a8e87, ASSIGN; state=OFFLINE, location=da1c6afcd1f9,34155,1731595913533; forceNewPlan=false, retain=false 2024-11-14T14:52:09,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42151 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-14T14:52:09,738 INFO [da1c6afcd1f9:42151 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-14T14:52:09,739 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=cf7ad4aad0d0e3c5a9f31c16500a8e87, regionState=OPENING, regionLocation=da1c6afcd1f9,34155,1731595913533 2024-11-14T14:52:09,742 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=cf7ad4aad0d0e3c5a9f31c16500a8e87, ASSIGN because future has completed 2024-11-14T14:52:09,743 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure cf7ad4aad0d0e3c5a9f31c16500a8e87, server=da1c6afcd1f9,34155,1731595913533}] 2024-11-14T14:52:09,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42151 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-14T14:52:09,895 DEBUG [RSProcedureDispatcher-pool-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-14T14:52:09,897 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49435, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-14T14:52:09,903 INFO [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1731595929525.cf7ad4aad0d0e3c5a9f31c16500a8e87. 2024-11-14T14:52:09,903 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => cf7ad4aad0d0e3c5a9f31c16500a8e87, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1731595929525.cf7ad4aad0d0e3c5a9f31c16500a8e87.', STARTKEY => '', ENDKEY => ''} 2024-11-14T14:52:09,904 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF cf7ad4aad0d0e3c5a9f31c16500a8e87 2024-11-14T14:52:09,904 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1731595929525.cf7ad4aad0d0e3c5a9f31c16500a8e87.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T14:52:09,904 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for cf7ad4aad0d0e3c5a9f31c16500a8e87 2024-11-14T14:52:09,904 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for cf7ad4aad0d0e3c5a9f31c16500a8e87 2024-11-14T14:52:09,906 INFO [StoreOpener-cf7ad4aad0d0e3c5a9f31c16500a8e87-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region cf7ad4aad0d0e3c5a9f31c16500a8e87 2024-11-14T14:52:09,908 INFO [StoreOpener-cf7ad4aad0d0e3c5a9f31c16500a8e87-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cf7ad4aad0d0e3c5a9f31c16500a8e87 columnFamilyName cf1 2024-11-14T14:52:09,908 DEBUG [StoreOpener-cf7ad4aad0d0e3c5a9f31c16500a8e87-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T14:52:09,908 INFO [StoreOpener-cf7ad4aad0d0e3c5a9f31c16500a8e87-1 {}] regionserver.HStore(327): Store=cf7ad4aad0d0e3c5a9f31c16500a8e87/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T14:52:09,908 INFO [StoreOpener-cf7ad4aad0d0e3c5a9f31c16500a8e87-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region cf7ad4aad0d0e3c5a9f31c16500a8e87 2024-11-14T14:52:09,910 INFO [StoreOpener-cf7ad4aad0d0e3c5a9f31c16500a8e87-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cf7ad4aad0d0e3c5a9f31c16500a8e87 columnFamilyName cf2 2024-11-14T14:52:09,910 DEBUG [StoreOpener-cf7ad4aad0d0e3c5a9f31c16500a8e87-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T14:52:09,910 INFO [StoreOpener-cf7ad4aad0d0e3c5a9f31c16500a8e87-1 {}] regionserver.HStore(327): Store=cf7ad4aad0d0e3c5a9f31c16500a8e87/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T14:52:09,910 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for cf7ad4aad0d0e3c5a9f31c16500a8e87 2024-11-14T14:52:09,911 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34445/user/jenkins/test-data/35161b07-018a-0344-8c43-1519f3812567/data/default/testReplayEditsAfterRegionMovedWithMultiCF/cf7ad4aad0d0e3c5a9f31c16500a8e87 2024-11-14T14:52:09,912 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34445/user/jenkins/test-data/35161b07-018a-0344-8c43-1519f3812567/data/default/testReplayEditsAfterRegionMovedWithMultiCF/cf7ad4aad0d0e3c5a9f31c16500a8e87 2024-11-14T14:52:09,913 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for cf7ad4aad0d0e3c5a9f31c16500a8e87 2024-11-14T14:52:09,913 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for cf7ad4aad0d0e3c5a9f31c16500a8e87 2024-11-14T14:52:09,914 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-11-14T14:52:09,916 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for cf7ad4aad0d0e3c5a9f31c16500a8e87 2024-11-14T14:52:09,919 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34445/user/jenkins/test-data/35161b07-018a-0344-8c43-1519f3812567/data/default/testReplayEditsAfterRegionMovedWithMultiCF/cf7ad4aad0d0e3c5a9f31c16500a8e87/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-14T14:52:09,919 INFO [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened cf7ad4aad0d0e3c5a9f31c16500a8e87; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59381592, jitterRate=-0.11514532566070557}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-11-14T14:52:09,919 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for cf7ad4aad0d0e3c5a9f31c16500a8e87 2024-11-14T14:52:09,920 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for cf7ad4aad0d0e3c5a9f31c16500a8e87: Running coprocessor pre-open hook at 1731595929905Writing region info on filesystem at 1731595929905Initializing all the Stores at 1731595929906 (+1 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731595929906Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731595929906Cleaning up temporary data from old regions at 1731595929913 (+7 ms)Running coprocessor post-open hooks at 1731595929919 (+6 ms)Region opened successfully at 1731595929920 (+1 ms) 2024-11-14T14:52:09,922 INFO [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1731595929525.cf7ad4aad0d0e3c5a9f31c16500a8e87., pid=6, masterSystemTime=1731595929895 2024-11-14T14:52:09,925 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1731595929525.cf7ad4aad0d0e3c5a9f31c16500a8e87. 2024-11-14T14:52:09,925 INFO [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1731595929525.cf7ad4aad0d0e3c5a9f31c16500a8e87. 2024-11-14T14:52:09,926 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=cf7ad4aad0d0e3c5a9f31c16500a8e87, regionState=OPEN, openSeqNum=2, regionLocation=da1c6afcd1f9,34155,1731595913533 2024-11-14T14:52:09,930 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure cf7ad4aad0d0e3c5a9f31c16500a8e87, server=da1c6afcd1f9,34155,1731595913533 because future has completed 2024-11-14T14:52:09,935 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-14T14:52:09,935 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure cf7ad4aad0d0e3c5a9f31c16500a8e87, server=da1c6afcd1f9,34155,1731595913533 in 188 msec 2024-11-14T14:52:09,938 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-14T14:52:09,938 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=cf7ad4aad0d0e3c5a9f31c16500a8e87, ASSIGN in 352 msec 2024-11-14T14:52:09,940 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-14T14:52:09,940 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testReplayEditsAfterRegionMovedWithMultiCF","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731595929940"}]},"ts":"1731595929940"} 2024-11-14T14:52:09,943 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testReplayEditsAfterRegionMovedWithMultiCF, state=ENABLED in hbase:meta 2024-11-14T14:52:09,944 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_POST_OPERATION 2024-11-14T14:52:09,947 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF in 414 msec 2024-11-14T14:52:10,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42151 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-14T14:52:10,176 INFO [RPCClient-NioEventLoopGroup-6-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testReplayEditsAfterRegionMovedWithMultiCF completed 2024-11-14T14:52:10,176 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testReplayEditsAfterRegionMovedWithMultiCF get assigned. Timeout = 60000ms 2024-11-14T14:52:10,178 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-14T14:52:10,183 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testReplayEditsAfterRegionMovedWithMultiCF assigned to meta. Checking AM states. 2024-11-14T14:52:10,184 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-14T14:52:10,184 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testReplayEditsAfterRegionMovedWithMultiCF assigned. 2024-11-14T14:52:10,197 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1731595929525.cf7ad4aad0d0e3c5a9f31c16500a8e87., hostname=da1c6afcd1f9,34155,1731595913533, seqNum=2] 2024-11-14T14:52:10,198 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T14:52:10,200 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36380, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T14:52:10,215 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42151 {}] master.HMaster(2410): Client=jenkins//172.17.0.2 move hri=cf7ad4aad0d0e3c5a9f31c16500a8e87, source=da1c6afcd1f9,34155,1731595913533, destination=da1c6afcd1f9,37695,1731595913427, warming up region on da1c6afcd1f9,37695,1731595913427 2024-11-14T14:52:10,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42151 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-14T14:52:10,218 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42151 {}] master.HMaster(2414): Client=jenkins//172.17.0.2 move hri=cf7ad4aad0d0e3c5a9f31c16500a8e87, source=da1c6afcd1f9,34155,1731595913533, destination=da1c6afcd1f9,37695,1731595913427, running balancer 2024-11-14T14:52:10,219 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52409, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-14T14:52:10,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42151 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=cf7ad4aad0d0e3c5a9f31c16500a8e87, REOPEN/MOVE 2024-11-14T14:52:10,220 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=cf7ad4aad0d0e3c5a9f31c16500a8e87, REOPEN/MOVE 2024-11-14T14:52:10,222 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=cf7ad4aad0d0e3c5a9f31c16500a8e87, regionState=CLOSING, regionLocation=da1c6afcd1f9,34155,1731595913533 2024-11-14T14:52:10,224 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] regionserver.RSRpcServices(2066): Warmup testReplayEditsAfterRegionMovedWithMultiCF,,1731595929525.cf7ad4aad0d0e3c5a9f31c16500a8e87. 2024-11-14T14:52:10,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] regionserver.HRegion(7855): Warmup {ENCODED => cf7ad4aad0d0e3c5a9f31c16500a8e87, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1731595929525.cf7ad4aad0d0e3c5a9f31c16500a8e87.', STARTKEY => '', ENDKEY => ''} 2024-11-14T14:52:10,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1731595929525.cf7ad4aad0d0e3c5a9f31c16500a8e87.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T14:52:10,225 INFO [StoreOpener-cf7ad4aad0d0e3c5a9f31c16500a8e87-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region cf7ad4aad0d0e3c5a9f31c16500a8e87 2024-11-14T14:52:10,225 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=cf7ad4aad0d0e3c5a9f31c16500a8e87, REOPEN/MOVE because future has completed 2024-11-14T14:52:10,226 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-14T14:52:10,226 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure cf7ad4aad0d0e3c5a9f31c16500a8e87, server=da1c6afcd1f9,34155,1731595913533}] 2024-11-14T14:52:10,227 INFO [StoreOpener-cf7ad4aad0d0e3c5a9f31c16500a8e87-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cf7ad4aad0d0e3c5a9f31c16500a8e87 columnFamilyName cf1 2024-11-14T14:52:10,227 DEBUG [StoreOpener-cf7ad4aad0d0e3c5a9f31c16500a8e87-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T14:52:10,227 INFO [StoreOpener-cf7ad4aad0d0e3c5a9f31c16500a8e87-1 {}] regionserver.HStore(327): Store=cf7ad4aad0d0e3c5a9f31c16500a8e87/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T14:52:10,228 INFO [StoreOpener-cf7ad4aad0d0e3c5a9f31c16500a8e87-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region cf7ad4aad0d0e3c5a9f31c16500a8e87 2024-11-14T14:52:10,229 INFO [StoreOpener-cf7ad4aad0d0e3c5a9f31c16500a8e87-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cf7ad4aad0d0e3c5a9f31c16500a8e87 columnFamilyName cf2 2024-11-14T14:52:10,229 DEBUG [StoreOpener-cf7ad4aad0d0e3c5a9f31c16500a8e87-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T14:52:10,230 INFO [StoreOpener-cf7ad4aad0d0e3c5a9f31c16500a8e87-1 {}] regionserver.HStore(327): Store=cf7ad4aad0d0e3c5a9f31c16500a8e87/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T14:52:10,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] regionserver.HRegion(1722): Closing cf7ad4aad0d0e3c5a9f31c16500a8e87, disabling compactions & flushes 2024-11-14T14:52:10,230 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1731595929525.cf7ad4aad0d0e3c5a9f31c16500a8e87. 2024-11-14T14:52:10,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731595929525.cf7ad4aad0d0e3c5a9f31c16500a8e87. 2024-11-14T14:52:10,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731595929525.cf7ad4aad0d0e3c5a9f31c16500a8e87. after waiting 0 ms 2024-11-14T14:52:10,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1731595929525.cf7ad4aad0d0e3c5a9f31c16500a8e87. 2024-11-14T14:52:10,232 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1731595929525.cf7ad4aad0d0e3c5a9f31c16500a8e87. 2024-11-14T14:52:10,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] regionserver.HRegion(1676): Region close journal for cf7ad4aad0d0e3c5a9f31c16500a8e87: Waiting for close lock at 1731595930230Disabling compacts and flushes for region at 1731595930230Disabling writes for close at 1731595930230Writing region close event to WAL at 1731595930232 (+2 ms)Closed at 1731595930232 2024-11-14T14:52:10,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42151 {}] procedure.ProcedureSyncWait(219): waitFor pid=7 2024-11-14T14:52:10,385 INFO [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(122): Close cf7ad4aad0d0e3c5a9f31c16500a8e87 2024-11-14T14:52:10,386 DEBUG [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-14T14:52:10,386 DEBUG [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1722): Closing cf7ad4aad0d0e3c5a9f31c16500a8e87, disabling compactions & flushes 2024-11-14T14:52:10,386 INFO [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1731595929525.cf7ad4aad0d0e3c5a9f31c16500a8e87. 2024-11-14T14:52:10,386 DEBUG [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731595929525.cf7ad4aad0d0e3c5a9f31c16500a8e87. 2024-11-14T14:52:10,386 DEBUG [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731595929525.cf7ad4aad0d0e3c5a9f31c16500a8e87. after waiting 0 ms 2024-11-14T14:52:10,387 DEBUG [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1731595929525.cf7ad4aad0d0e3c5a9f31c16500a8e87. 2024-11-14T14:52:10,387 INFO [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(2902): Flushing cf7ad4aad0d0e3c5a9f31c16500a8e87 2/2 column families, dataSize=31 B heapSize=616 B 2024-11-14T14:52:10,406 DEBUG [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/35161b07-018a-0344-8c43-1519f3812567/data/default/testReplayEditsAfterRegionMovedWithMultiCF/cf7ad4aad0d0e3c5a9f31c16500a8e87/.tmp/cf1/cf87dfececad472fa7235bfb764bd7d6 is 35, key is r1/cf1:q/1731595930201/Put/seqid=0 2024-11-14T14:52:10,414 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36189 is added to blk_1073741887_1065 (size=4783) 2024-11-14T14:52:10,414 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44549 is added to blk_1073741887_1065 (size=4783) 2024-11-14T14:52:10,414 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42819 is added to blk_1073741887_1065 (size=4783) 2024-11-14T14:52:10,415 INFO [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31 B at sequenceid=5 (bloomFilter=false), to=hdfs://localhost:34445/user/jenkins/test-data/35161b07-018a-0344-8c43-1519f3812567/data/default/testReplayEditsAfterRegionMovedWithMultiCF/cf7ad4aad0d0e3c5a9f31c16500a8e87/.tmp/cf1/cf87dfececad472fa7235bfb764bd7d6 2024-11-14T14:52:10,423 DEBUG [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/35161b07-018a-0344-8c43-1519f3812567/data/default/testReplayEditsAfterRegionMovedWithMultiCF/cf7ad4aad0d0e3c5a9f31c16500a8e87/.tmp/cf1/cf87dfececad472fa7235bfb764bd7d6 as hdfs://localhost:34445/user/jenkins/test-data/35161b07-018a-0344-8c43-1519f3812567/data/default/testReplayEditsAfterRegionMovedWithMultiCF/cf7ad4aad0d0e3c5a9f31c16500a8e87/cf1/cf87dfececad472fa7235bfb764bd7d6 2024-11-14T14:52:10,429 INFO [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/35161b07-018a-0344-8c43-1519f3812567/data/default/testReplayEditsAfterRegionMovedWithMultiCF/cf7ad4aad0d0e3c5a9f31c16500a8e87/cf1/cf87dfececad472fa7235bfb764bd7d6, entries=1, sequenceid=5, filesize=4.7 K 2024-11-14T14:52:10,431 INFO [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~31 B/31, heapSize ~344 B/344, currentSize=0 B/0 for cf7ad4aad0d0e3c5a9f31c16500a8e87 in 44ms, sequenceid=5, compaction requested=false 2024-11-14T14:52:10,431 DEBUG [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testReplayEditsAfterRegionMovedWithMultiCF' 2024-11-14T14:52:10,436 DEBUG [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34445/user/jenkins/test-data/35161b07-018a-0344-8c43-1519f3812567/data/default/testReplayEditsAfterRegionMovedWithMultiCF/cf7ad4aad0d0e3c5a9f31c16500a8e87/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-14T14:52:10,440 INFO [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1731595929525.cf7ad4aad0d0e3c5a9f31c16500a8e87. 2024-11-14T14:52:10,440 DEBUG [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1676): Region close journal for cf7ad4aad0d0e3c5a9f31c16500a8e87: Waiting for close lock at 1731595930386Running coprocessor pre-close hooks at 1731595930386Disabling compacts and flushes for region at 1731595930386Disabling writes for close at 1731595930386Obtaining lock to block concurrent updates at 1731595930387 (+1 ms)Preparing flush snapshotting stores in cf7ad4aad0d0e3c5a9f31c16500a8e87 at 1731595930387Finished memstore snapshotting testReplayEditsAfterRegionMovedWithMultiCF,,1731595929525.cf7ad4aad0d0e3c5a9f31c16500a8e87., syncing WAL and waiting on mvcc, flushsize=dataSize=31, getHeapSize=584, getOffHeapSize=0, getCellsCount=1 at 1731595930387Flushing stores of testReplayEditsAfterRegionMovedWithMultiCF,,1731595929525.cf7ad4aad0d0e3c5a9f31c16500a8e87. at 1731595930388 (+1 ms)Flushing cf7ad4aad0d0e3c5a9f31c16500a8e87/cf1: creating writer at 1731595930388Flushing cf7ad4aad0d0e3c5a9f31c16500a8e87/cf1: appending metadata at 1731595930405 (+17 ms)Flushing cf7ad4aad0d0e3c5a9f31c16500a8e87/cf1: closing flushed file at 1731595930405Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@160e388f: reopening flushed file at 1731595930422 (+17 ms)Finished flush of dataSize ~31 B/31, heapSize ~344 B/344, currentSize=0 B/0 for cf7ad4aad0d0e3c5a9f31c16500a8e87 in 44ms, sequenceid=5, compaction requested=false at 1731595930431 (+9 ms)Writing region close event to WAL at 1731595930432 (+1 ms)Running coprocessor post-close hooks at 1731595930437 (+5 ms)Closed at 1731595930440 (+3 ms) 2024-11-14T14:52:10,440 INFO [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionServer(3302): Adding cf7ad4aad0d0e3c5a9f31c16500a8e87 move to da1c6afcd1f9,37695,1731595913427 record at close sequenceid=5 2024-11-14T14:52:10,444 INFO [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(157): Closed cf7ad4aad0d0e3c5a9f31c16500a8e87 2024-11-14T14:52:10,445 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=cf7ad4aad0d0e3c5a9f31c16500a8e87, regionState=CLOSED 2024-11-14T14:52:10,448 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure cf7ad4aad0d0e3c5a9f31c16500a8e87, server=da1c6afcd1f9,34155,1731595913533 because future has completed 2024-11-14T14:52:10,453 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-14T14:52:10,453 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; CloseRegionProcedure cf7ad4aad0d0e3c5a9f31c16500a8e87, server=da1c6afcd1f9,34155,1731595913533 in 223 msec 2024-11-14T14:52:10,454 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=cf7ad4aad0d0e3c5a9f31c16500a8e87, REOPEN/MOVE; state=CLOSED, location=da1c6afcd1f9,37695,1731595913427; forceNewPlan=false, retain=false 2024-11-14T14:52:10,604 INFO [da1c6afcd1f9:42151 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-14T14:52:10,605 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=cf7ad4aad0d0e3c5a9f31c16500a8e87, regionState=OPENING, regionLocation=da1c6afcd1f9,37695,1731595913427 2024-11-14T14:52:10,608 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=cf7ad4aad0d0e3c5a9f31c16500a8e87, REOPEN/MOVE because future has completed 2024-11-14T14:52:10,608 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure cf7ad4aad0d0e3c5a9f31c16500a8e87, server=da1c6afcd1f9,37695,1731595913427}] 2024-11-14T14:52:10,767 INFO [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1731595929525.cf7ad4aad0d0e3c5a9f31c16500a8e87. 2024-11-14T14:52:10,767 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => cf7ad4aad0d0e3c5a9f31c16500a8e87, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1731595929525.cf7ad4aad0d0e3c5a9f31c16500a8e87.', STARTKEY => '', ENDKEY => ''} 2024-11-14T14:52:10,767 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF cf7ad4aad0d0e3c5a9f31c16500a8e87 2024-11-14T14:52:10,767 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1731595929525.cf7ad4aad0d0e3c5a9f31c16500a8e87.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T14:52:10,768 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for cf7ad4aad0d0e3c5a9f31c16500a8e87 2024-11-14T14:52:10,768 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for cf7ad4aad0d0e3c5a9f31c16500a8e87 2024-11-14T14:52:10,769 INFO [StoreOpener-cf7ad4aad0d0e3c5a9f31c16500a8e87-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region cf7ad4aad0d0e3c5a9f31c16500a8e87 2024-11-14T14:52:10,770 INFO [StoreOpener-cf7ad4aad0d0e3c5a9f31c16500a8e87-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cf7ad4aad0d0e3c5a9f31c16500a8e87 columnFamilyName cf1 2024-11-14T14:52:10,770 DEBUG [StoreOpener-cf7ad4aad0d0e3c5a9f31c16500a8e87-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T14:52:10,777 DEBUG [StoreOpener-cf7ad4aad0d0e3c5a9f31c16500a8e87-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:34445/user/jenkins/test-data/35161b07-018a-0344-8c43-1519f3812567/data/default/testReplayEditsAfterRegionMovedWithMultiCF/cf7ad4aad0d0e3c5a9f31c16500a8e87/cf1/cf87dfececad472fa7235bfb764bd7d6 2024-11-14T14:52:10,777 INFO [StoreOpener-cf7ad4aad0d0e3c5a9f31c16500a8e87-1 {}] regionserver.HStore(327): Store=cf7ad4aad0d0e3c5a9f31c16500a8e87/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T14:52:10,777 INFO [StoreOpener-cf7ad4aad0d0e3c5a9f31c16500a8e87-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region cf7ad4aad0d0e3c5a9f31c16500a8e87 2024-11-14T14:52:10,778 INFO [StoreOpener-cf7ad4aad0d0e3c5a9f31c16500a8e87-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cf7ad4aad0d0e3c5a9f31c16500a8e87 columnFamilyName cf2 2024-11-14T14:52:10,778 DEBUG [StoreOpener-cf7ad4aad0d0e3c5a9f31c16500a8e87-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T14:52:10,778 INFO [StoreOpener-cf7ad4aad0d0e3c5a9f31c16500a8e87-1 {}] regionserver.HStore(327): Store=cf7ad4aad0d0e3c5a9f31c16500a8e87/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T14:52:10,779 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for cf7ad4aad0d0e3c5a9f31c16500a8e87 2024-11-14T14:52:10,780 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34445/user/jenkins/test-data/35161b07-018a-0344-8c43-1519f3812567/data/default/testReplayEditsAfterRegionMovedWithMultiCF/cf7ad4aad0d0e3c5a9f31c16500a8e87 2024-11-14T14:52:10,781 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34445/user/jenkins/test-data/35161b07-018a-0344-8c43-1519f3812567/data/default/testReplayEditsAfterRegionMovedWithMultiCF/cf7ad4aad0d0e3c5a9f31c16500a8e87 2024-11-14T14:52:10,782 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for cf7ad4aad0d0e3c5a9f31c16500a8e87 2024-11-14T14:52:10,782 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for cf7ad4aad0d0e3c5a9f31c16500a8e87 2024-11-14T14:52:10,783 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-11-14T14:52:10,784 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for cf7ad4aad0d0e3c5a9f31c16500a8e87 2024-11-14T14:52:10,785 INFO [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1114): Opened cf7ad4aad0d0e3c5a9f31c16500a8e87; next sequenceid=9; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73711208, jitterRate=0.09838259220123291}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-11-14T14:52:10,785 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for cf7ad4aad0d0e3c5a9f31c16500a8e87 2024-11-14T14:52:10,786 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for cf7ad4aad0d0e3c5a9f31c16500a8e87: Running coprocessor pre-open hook at 1731595930768Writing region info on filesystem at 1731595930768Initializing all the Stores at 1731595930769 (+1 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731595930769Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731595930769Cleaning up temporary data from old regions at 1731595930782 (+13 ms)Running coprocessor post-open hooks at 1731595930785 (+3 ms)Region opened successfully at 1731595930786 (+1 ms) 2024-11-14T14:52:10,788 INFO [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1731595929525.cf7ad4aad0d0e3c5a9f31c16500a8e87., pid=9, masterSystemTime=1731595930762 2024-11-14T14:52:10,790 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1731595929525.cf7ad4aad0d0e3c5a9f31c16500a8e87. 2024-11-14T14:52:10,790 INFO [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1731595929525.cf7ad4aad0d0e3c5a9f31c16500a8e87. 2024-11-14T14:52:10,791 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=cf7ad4aad0d0e3c5a9f31c16500a8e87, regionState=OPEN, openSeqNum=9, regionLocation=da1c6afcd1f9,37695,1731595913427 2024-11-14T14:52:10,794 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure cf7ad4aad0d0e3c5a9f31c16500a8e87, server=da1c6afcd1f9,37695,1731595913427 because future has completed 2024-11-14T14:52:10,798 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=7 2024-11-14T14:52:10,798 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=7, state=SUCCESS, hasLock=false; OpenRegionProcedure cf7ad4aad0d0e3c5a9f31c16500a8e87, server=da1c6afcd1f9,37695,1731595913427 in 187 msec 2024-11-14T14:52:10,800 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=cf7ad4aad0d0e3c5a9f31c16500a8e87, REOPEN/MOVE in 580 msec 2024-11-14T14:52:10,826 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-14T14:52:10,828 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43350, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-14T14:52:10,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34155 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 139 connection: 172.17.0.2:36380 deadline: 1731595990832, exception=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=da1c6afcd1f9 port=37695 startCode=1731595913427. As of locationSeqNum=5. 2024-11-14T14:52:10,839 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1731595929525.cf7ad4aad0d0e3c5a9f31c16500a8e87., hostname=da1c6afcd1f9,34155,1731595913533, seqNum=2 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1731595929525.cf7ad4aad0d0e3c5a9f31c16500a8e87., hostname=da1c6afcd1f9,34155,1731595913533, seqNum=2, error=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=da1c6afcd1f9 port=37695 startCode=1731595913427. As of locationSeqNum=5. 2024-11-14T14:52:10,839 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1731595929525.cf7ad4aad0d0e3c5a9f31c16500a8e87., hostname=da1c6afcd1f9,34155,1731595913533, seqNum=2 is org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=da1c6afcd1f9 port=37695 startCode=1731595913427. As of locationSeqNum=5. 2024-11-14T14:52:10,839 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(84): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1731595929525.cf7ad4aad0d0e3c5a9f31c16500a8e87., hostname=da1c6afcd1f9,34155,1731595913533, seqNum=2 with the new location region=testReplayEditsAfterRegionMovedWithMultiCF,,1731595929525.cf7ad4aad0d0e3c5a9f31c16500a8e87., hostname=da1c6afcd1f9,37695,1731595913427, seqNum=5 constructed by org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=da1c6afcd1f9 port=37695 startCode=1731595913427. As of locationSeqNum=5. 2024-11-14T14:52:10,945 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T14:52:10,948 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43358, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T14:52:10,958 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing cf7ad4aad0d0e3c5a9f31c16500a8e87 2/2 column families, dataSize=50 B heapSize=720 B 2024-11-14T14:52:10,975 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/35161b07-018a-0344-8c43-1519f3812567/data/default/testReplayEditsAfterRegionMovedWithMultiCF/cf7ad4aad0d0e3c5a9f31c16500a8e87/.tmp/cf1/f179df72d6e5438c8c11097ad04e717e is 29, key is r1/cf1:/1731595930949/DeleteFamily/seqid=0 2024-11-14T14:52:10,982 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44549 is added to blk_1073741888_1066 (size=4906) 2024-11-14T14:52:10,982 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42819 is added to blk_1073741888_1066 (size=4906) 2024-11-14T14:52:10,983 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36189 is added to blk_1073741888_1066 (size=4906) 2024-11-14T14:52:10,983 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=25 B at sequenceid=12 (bloomFilter=false), to=hdfs://localhost:34445/user/jenkins/test-data/35161b07-018a-0344-8c43-1519f3812567/data/default/testReplayEditsAfterRegionMovedWithMultiCF/cf7ad4aad0d0e3c5a9f31c16500a8e87/.tmp/cf1/f179df72d6e5438c8c11097ad04e717e 2024-11-14T14:52:10,990 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for f179df72d6e5438c8c11097ad04e717e 2024-11-14T14:52:11,004 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/35161b07-018a-0344-8c43-1519f3812567/data/default/testReplayEditsAfterRegionMovedWithMultiCF/cf7ad4aad0d0e3c5a9f31c16500a8e87/.tmp/cf2/7be00c9408c842239946f9a0bad594bd is 29, key is r1/cf2:/1731595930949/DeleteFamily/seqid=0 2024-11-14T14:52:11,010 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44549 is added to blk_1073741889_1067 (size=4906) 2024-11-14T14:52:11,011 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42819 is added to blk_1073741889_1067 (size=4906) 2024-11-14T14:52:11,011 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36189 is added to blk_1073741889_1067 (size=4906) 2024-11-14T14:52:11,011 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=25 B at sequenceid=12 (bloomFilter=false), to=hdfs://localhost:34445/user/jenkins/test-data/35161b07-018a-0344-8c43-1519f3812567/data/default/testReplayEditsAfterRegionMovedWithMultiCF/cf7ad4aad0d0e3c5a9f31c16500a8e87/.tmp/cf2/7be00c9408c842239946f9a0bad594bd 2024-11-14T14:52:11,017 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 7be00c9408c842239946f9a0bad594bd 2024-11-14T14:52:11,018 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/35161b07-018a-0344-8c43-1519f3812567/data/default/testReplayEditsAfterRegionMovedWithMultiCF/cf7ad4aad0d0e3c5a9f31c16500a8e87/.tmp/cf1/f179df72d6e5438c8c11097ad04e717e as hdfs://localhost:34445/user/jenkins/test-data/35161b07-018a-0344-8c43-1519f3812567/data/default/testReplayEditsAfterRegionMovedWithMultiCF/cf7ad4aad0d0e3c5a9f31c16500a8e87/cf1/f179df72d6e5438c8c11097ad04e717e 2024-11-14T14:52:11,023 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for f179df72d6e5438c8c11097ad04e717e 2024-11-14T14:52:11,023 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/35161b07-018a-0344-8c43-1519f3812567/data/default/testReplayEditsAfterRegionMovedWithMultiCF/cf7ad4aad0d0e3c5a9f31c16500a8e87/cf1/f179df72d6e5438c8c11097ad04e717e, entries=1, sequenceid=12, filesize=4.8 K 2024-11-14T14:52:11,024 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/35161b07-018a-0344-8c43-1519f3812567/data/default/testReplayEditsAfterRegionMovedWithMultiCF/cf7ad4aad0d0e3c5a9f31c16500a8e87/.tmp/cf2/7be00c9408c842239946f9a0bad594bd as hdfs://localhost:34445/user/jenkins/test-data/35161b07-018a-0344-8c43-1519f3812567/data/default/testReplayEditsAfterRegionMovedWithMultiCF/cf7ad4aad0d0e3c5a9f31c16500a8e87/cf2/7be00c9408c842239946f9a0bad594bd 2024-11-14T14:52:11,029 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 7be00c9408c842239946f9a0bad594bd 2024-11-14T14:52:11,030 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/35161b07-018a-0344-8c43-1519f3812567/data/default/testReplayEditsAfterRegionMovedWithMultiCF/cf7ad4aad0d0e3c5a9f31c16500a8e87/cf2/7be00c9408c842239946f9a0bad594bd, entries=1, sequenceid=12, filesize=4.8 K 2024-11-14T14:52:11,031 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~50 B/50, heapSize ~688 B/688, currentSize=0 B/0 for cf7ad4aad0d0e3c5a9f31c16500a8e87 in 72ms, sequenceid=12, compaction requested=false 2024-11-14T14:52:11,031 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for cf7ad4aad0d0e3c5a9f31c16500a8e87: 2024-11-14T14:52:11,033 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-14T14:52:11,035 DEBUG [Time-limited test {}] regionserver.HStore(1541): cf7ad4aad0d0e3c5a9f31c16500a8e87/cf1 is initiating major compaction (all files) 2024-11-14T14:52:11,035 INFO [Time-limited test {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-14T14:52:11,035 INFO [Time-limited test {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-14T14:52:11,035 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of cf7ad4aad0d0e3c5a9f31c16500a8e87/cf1 in testReplayEditsAfterRegionMovedWithMultiCF,,1731595929525.cf7ad4aad0d0e3c5a9f31c16500a8e87. 2024-11-14T14:52:11,036 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34445/user/jenkins/test-data/35161b07-018a-0344-8c43-1519f3812567/data/default/testReplayEditsAfterRegionMovedWithMultiCF/cf7ad4aad0d0e3c5a9f31c16500a8e87/cf1/cf87dfececad472fa7235bfb764bd7d6, hdfs://localhost:34445/user/jenkins/test-data/35161b07-018a-0344-8c43-1519f3812567/data/default/testReplayEditsAfterRegionMovedWithMultiCF/cf7ad4aad0d0e3c5a9f31c16500a8e87/cf1/f179df72d6e5438c8c11097ad04e717e] into tmpdir=hdfs://localhost:34445/user/jenkins/test-data/35161b07-018a-0344-8c43-1519f3812567/data/default/testReplayEditsAfterRegionMovedWithMultiCF/cf7ad4aad0d0e3c5a9f31c16500a8e87/.tmp, totalSize=9.5 K 2024-11-14T14:52:11,036 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting cf87dfececad472fa7235bfb764bd7d6, keycount=1, bloomtype=NONE, size=4.7 K, encoding=NONE, compression=NONE, seqNum=5, earliestPutTs=1731595930201 2024-11-14T14:52:11,037 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting f179df72d6e5438c8c11097ad04e717e, keycount=1, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=9223372036854775807 2024-11-14T14:52:11,046 INFO [Time-limited test {}] throttle.PressureAwareThroughputController(145): cf7ad4aad0d0e3c5a9f31c16500a8e87#cf1#compaction#16 average throughput is NaN MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T14:52:11,053 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42819 is added to blk_1073741890_1068 (size=4626) 2024-11-14T14:52:11,053 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36189 is added to blk_1073741890_1068 (size=4626) 2024-11-14T14:52:11,054 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44549 is added to blk_1073741890_1068 (size=4626) 2024-11-14T14:52:11,060 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/35161b07-018a-0344-8c43-1519f3812567/data/default/testReplayEditsAfterRegionMovedWithMultiCF/cf7ad4aad0d0e3c5a9f31c16500a8e87/.tmp/cf1/efdb7fed42594e78b882b0ad2e13a8c1 as hdfs://localhost:34445/user/jenkins/test-data/35161b07-018a-0344-8c43-1519f3812567/data/default/testReplayEditsAfterRegionMovedWithMultiCF/cf7ad4aad0d0e3c5a9f31c16500a8e87/cf1/efdb7fed42594e78b882b0ad2e13a8c1 2024-11-14T14:52:11,074 INFO [Time-limited test {}] regionserver.HStore(1337): Completed major compaction of 2 (all) file(s) in cf7ad4aad0d0e3c5a9f31c16500a8e87/cf1 of cf7ad4aad0d0e3c5a9f31c16500a8e87 into efdb7fed42594e78b882b0ad2e13a8c1(size=4.5 K), total size for store is 4.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T14:52:11,074 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for cf7ad4aad0d0e3c5a9f31c16500a8e87: 2024-11-14T14:52:11,074 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 1 store files, 0 compacting, 1 eligible, 16 blocking 2024-11-14T14:52:11,074 DEBUG [Time-limited test {}] regionserver.HStore(1541): cf7ad4aad0d0e3c5a9f31c16500a8e87/cf2 is initiating major compaction (all files) 2024-11-14T14:52:11,075 INFO [Time-limited test {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-14T14:52:11,075 INFO [Time-limited test {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-14T14:52:11,075 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of cf7ad4aad0d0e3c5a9f31c16500a8e87/cf2 in testReplayEditsAfterRegionMovedWithMultiCF,,1731595929525.cf7ad4aad0d0e3c5a9f31c16500a8e87. 2024-11-14T14:52:11,075 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34445/user/jenkins/test-data/35161b07-018a-0344-8c43-1519f3812567/data/default/testReplayEditsAfterRegionMovedWithMultiCF/cf7ad4aad0d0e3c5a9f31c16500a8e87/cf2/7be00c9408c842239946f9a0bad594bd] into tmpdir=hdfs://localhost:34445/user/jenkins/test-data/35161b07-018a-0344-8c43-1519f3812567/data/default/testReplayEditsAfterRegionMovedWithMultiCF/cf7ad4aad0d0e3c5a9f31c16500a8e87/.tmp, totalSize=4.8 K 2024-11-14T14:52:11,076 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 7be00c9408c842239946f9a0bad594bd, keycount=1, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=9223372036854775807 2024-11-14T14:52:11,081 INFO [Time-limited test {}] throttle.PressureAwareThroughputController(145): cf7ad4aad0d0e3c5a9f31c16500a8e87#cf2#compaction#17 average throughput is NaN MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T14:52:11,088 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44549 is added to blk_1073741891_1069 (size=4592) 2024-11-14T14:52:11,088 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42819 is added to blk_1073741891_1069 (size=4592) 2024-11-14T14:52:11,088 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36189 is added to blk_1073741891_1069 (size=4592) 2024-11-14T14:52:11,095 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/35161b07-018a-0344-8c43-1519f3812567/data/default/testReplayEditsAfterRegionMovedWithMultiCF/cf7ad4aad0d0e3c5a9f31c16500a8e87/.tmp/cf2/2b8c1aa97dce4f2aa88a5be7c980475c as hdfs://localhost:34445/user/jenkins/test-data/35161b07-018a-0344-8c43-1519f3812567/data/default/testReplayEditsAfterRegionMovedWithMultiCF/cf7ad4aad0d0e3c5a9f31c16500a8e87/cf2/2b8c1aa97dce4f2aa88a5be7c980475c 2024-11-14T14:52:11,102 INFO [Time-limited test {}] regionserver.HStore(1337): Completed major compaction of 1 (all) file(s) in cf7ad4aad0d0e3c5a9f31c16500a8e87/cf2 of cf7ad4aad0d0e3c5a9f31c16500a8e87 into 2b8c1aa97dce4f2aa88a5be7c980475c(size=4.5 K), total size for store is 4.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T14:52:11,102 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for cf7ad4aad0d0e3c5a9f31c16500a8e87: 2024-11-14T14:52:11,106 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42151 {}] master.HMaster(2410): Client=jenkins//172.17.0.2 move hri=cf7ad4aad0d0e3c5a9f31c16500a8e87, source=da1c6afcd1f9,37695,1731595913427, destination=da1c6afcd1f9,34155,1731595913533, warming up region on da1c6afcd1f9,34155,1731595913533 2024-11-14T14:52:11,106 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42151 {}] master.HMaster(2414): Client=jenkins//172.17.0.2 move hri=cf7ad4aad0d0e3c5a9f31c16500a8e87, source=da1c6afcd1f9,37695,1731595913427, destination=da1c6afcd1f9,34155,1731595913533, running balancer 2024-11-14T14:52:11,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42151 {}] procedure2.ProcedureExecutor(1139): Stored pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=cf7ad4aad0d0e3c5a9f31c16500a8e87, REOPEN/MOVE 2024-11-14T14:52:11,108 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=cf7ad4aad0d0e3c5a9f31c16500a8e87, REOPEN/MOVE 2024-11-14T14:52:11,109 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=cf7ad4aad0d0e3c5a9f31c16500a8e87, regionState=CLOSING, regionLocation=da1c6afcd1f9,37695,1731595913427 2024-11-14T14:52:11,109 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34155 {}] regionserver.RSRpcServices(2066): Warmup testReplayEditsAfterRegionMovedWithMultiCF,,1731595929525.cf7ad4aad0d0e3c5a9f31c16500a8e87. 2024-11-14T14:52:11,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34155 {}] regionserver.HRegion(7855): Warmup {ENCODED => cf7ad4aad0d0e3c5a9f31c16500a8e87, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1731595929525.cf7ad4aad0d0e3c5a9f31c16500a8e87.', STARTKEY => '', ENDKEY => ''} 2024-11-14T14:52:11,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34155 {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1731595929525.cf7ad4aad0d0e3c5a9f31c16500a8e87.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T14:52:11,111 INFO [StoreOpener-cf7ad4aad0d0e3c5a9f31c16500a8e87-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region cf7ad4aad0d0e3c5a9f31c16500a8e87 2024-11-14T14:52:11,112 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=cf7ad4aad0d0e3c5a9f31c16500a8e87, REOPEN/MOVE because future has completed 2024-11-14T14:52:11,112 INFO [StoreOpener-cf7ad4aad0d0e3c5a9f31c16500a8e87-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cf7ad4aad0d0e3c5a9f31c16500a8e87 columnFamilyName cf1 2024-11-14T14:52:11,112 DEBUG [StoreOpener-cf7ad4aad0d0e3c5a9f31c16500a8e87-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T14:52:11,112 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-14T14:52:11,112 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=11, ppid=10, state=RUNNABLE, hasLock=false; CloseRegionProcedure cf7ad4aad0d0e3c5a9f31c16500a8e87, server=da1c6afcd1f9,37695,1731595913427}] 2024-11-14T14:52:11,120 DEBUG [StoreOpener-cf7ad4aad0d0e3c5a9f31c16500a8e87-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:34445/user/jenkins/test-data/35161b07-018a-0344-8c43-1519f3812567/data/default/testReplayEditsAfterRegionMovedWithMultiCF/cf7ad4aad0d0e3c5a9f31c16500a8e87/cf1/cf87dfececad472fa7235bfb764bd7d6 2024-11-14T14:52:11,124 DEBUG [StoreOpener-cf7ad4aad0d0e3c5a9f31c16500a8e87-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:34445/user/jenkins/test-data/35161b07-018a-0344-8c43-1519f3812567/data/default/testReplayEditsAfterRegionMovedWithMultiCF/cf7ad4aad0d0e3c5a9f31c16500a8e87/cf1/efdb7fed42594e78b882b0ad2e13a8c1 2024-11-14T14:52:11,129 INFO [StoreFileOpener-cf7ad4aad0d0e3c5a9f31c16500a8e87-cf1-1 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for f179df72d6e5438c8c11097ad04e717e 2024-11-14T14:52:11,130 DEBUG [StoreOpener-cf7ad4aad0d0e3c5a9f31c16500a8e87-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:34445/user/jenkins/test-data/35161b07-018a-0344-8c43-1519f3812567/data/default/testReplayEditsAfterRegionMovedWithMultiCF/cf7ad4aad0d0e3c5a9f31c16500a8e87/cf1/f179df72d6e5438c8c11097ad04e717e 2024-11-14T14:52:11,130 INFO [StoreOpener-cf7ad4aad0d0e3c5a9f31c16500a8e87-1 {}] regionserver.HStore(327): Store=cf7ad4aad0d0e3c5a9f31c16500a8e87/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T14:52:11,130 INFO [StoreOpener-cf7ad4aad0d0e3c5a9f31c16500a8e87-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region cf7ad4aad0d0e3c5a9f31c16500a8e87 2024-11-14T14:52:11,131 INFO [StoreOpener-cf7ad4aad0d0e3c5a9f31c16500a8e87-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cf7ad4aad0d0e3c5a9f31c16500a8e87 columnFamilyName cf2 2024-11-14T14:52:11,131 DEBUG [StoreOpener-cf7ad4aad0d0e3c5a9f31c16500a8e87-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T14:52:11,137 DEBUG [StoreOpener-cf7ad4aad0d0e3c5a9f31c16500a8e87-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:34445/user/jenkins/test-data/35161b07-018a-0344-8c43-1519f3812567/data/default/testReplayEditsAfterRegionMovedWithMultiCF/cf7ad4aad0d0e3c5a9f31c16500a8e87/cf2/2b8c1aa97dce4f2aa88a5be7c980475c 2024-11-14T14:52:11,141 INFO [StoreFileOpener-cf7ad4aad0d0e3c5a9f31c16500a8e87-cf2-1 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 7be00c9408c842239946f9a0bad594bd 2024-11-14T14:52:11,141 DEBUG [StoreOpener-cf7ad4aad0d0e3c5a9f31c16500a8e87-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:34445/user/jenkins/test-data/35161b07-018a-0344-8c43-1519f3812567/data/default/testReplayEditsAfterRegionMovedWithMultiCF/cf7ad4aad0d0e3c5a9f31c16500a8e87/cf2/7be00c9408c842239946f9a0bad594bd 2024-11-14T14:52:11,142 INFO [StoreOpener-cf7ad4aad0d0e3c5a9f31c16500a8e87-1 {}] regionserver.HStore(327): Store=cf7ad4aad0d0e3c5a9f31c16500a8e87/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T14:52:11,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34155 {}] regionserver.HRegion(1722): Closing cf7ad4aad0d0e3c5a9f31c16500a8e87, disabling compactions & flushes 2024-11-14T14:52:11,142 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34155 {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1731595929525.cf7ad4aad0d0e3c5a9f31c16500a8e87. 2024-11-14T14:52:11,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34155 {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731595929525.cf7ad4aad0d0e3c5a9f31c16500a8e87. 2024-11-14T14:52:11,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34155 {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731595929525.cf7ad4aad0d0e3c5a9f31c16500a8e87. after waiting 0 ms 2024-11-14T14:52:11,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34155 {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1731595929525.cf7ad4aad0d0e3c5a9f31c16500a8e87. 2024-11-14T14:52:11,143 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34155 {}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1731595929525.cf7ad4aad0d0e3c5a9f31c16500a8e87. 2024-11-14T14:52:11,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34155 {}] regionserver.HRegion(1676): Region close journal for cf7ad4aad0d0e3c5a9f31c16500a8e87: Waiting for close lock at 1731595931142Disabling compacts and flushes for region at 1731595931142Disabling writes for close at 1731595931142Writing region close event to WAL at 1731595931143 (+1 ms)Closed at 1731595931143 2024-11-14T14:52:11,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42151 {}] procedure.ProcedureSyncWait(219): waitFor pid=10 2024-11-14T14:52:11,266 INFO [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] handler.UnassignRegionHandler(122): Close cf7ad4aad0d0e3c5a9f31c16500a8e87 2024-11-14T14:52:11,266 DEBUG [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-14T14:52:11,266 DEBUG [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1722): Closing cf7ad4aad0d0e3c5a9f31c16500a8e87, disabling compactions & flushes 2024-11-14T14:52:11,266 INFO [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1731595929525.cf7ad4aad0d0e3c5a9f31c16500a8e87. 2024-11-14T14:52:11,267 DEBUG [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731595929525.cf7ad4aad0d0e3c5a9f31c16500a8e87. 2024-11-14T14:52:11,267 DEBUG [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731595929525.cf7ad4aad0d0e3c5a9f31c16500a8e87. after waiting 0 ms 2024-11-14T14:52:11,267 DEBUG [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1731595929525.cf7ad4aad0d0e3c5a9f31c16500a8e87. 2024-11-14T14:52:11,267 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1731595929525.cf7ad4aad0d0e3c5a9f31c16500a8e87.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34445/user/jenkins/test-data/35161b07-018a-0344-8c43-1519f3812567/data/default/testReplayEditsAfterRegionMovedWithMultiCF/cf7ad4aad0d0e3c5a9f31c16500a8e87/cf1/cf87dfececad472fa7235bfb764bd7d6, hdfs://localhost:34445/user/jenkins/test-data/35161b07-018a-0344-8c43-1519f3812567/data/default/testReplayEditsAfterRegionMovedWithMultiCF/cf7ad4aad0d0e3c5a9f31c16500a8e87/cf1/f179df72d6e5438c8c11097ad04e717e] to archive 2024-11-14T14:52:11,270 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1731595929525.cf7ad4aad0d0e3c5a9f31c16500a8e87.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-14T14:52:11,274 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1731595929525.cf7ad4aad0d0e3c5a9f31c16500a8e87.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/35161b07-018a-0344-8c43-1519f3812567/data/default/testReplayEditsAfterRegionMovedWithMultiCF/cf7ad4aad0d0e3c5a9f31c16500a8e87/cf1/cf87dfececad472fa7235bfb764bd7d6 to hdfs://localhost:34445/user/jenkins/test-data/35161b07-018a-0344-8c43-1519f3812567/archive/data/default/testReplayEditsAfterRegionMovedWithMultiCF/cf7ad4aad0d0e3c5a9f31c16500a8e87/cf1/cf87dfececad472fa7235bfb764bd7d6 2024-11-14T14:52:11,276 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1731595929525.cf7ad4aad0d0e3c5a9f31c16500a8e87.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/35161b07-018a-0344-8c43-1519f3812567/data/default/testReplayEditsAfterRegionMovedWithMultiCF/cf7ad4aad0d0e3c5a9f31c16500a8e87/cf1/f179df72d6e5438c8c11097ad04e717e to hdfs://localhost:34445/user/jenkins/test-data/35161b07-018a-0344-8c43-1519f3812567/archive/data/default/testReplayEditsAfterRegionMovedWithMultiCF/cf7ad4aad0d0e3c5a9f31c16500a8e87/cf1/f179df72d6e5438c8c11097ad04e717e 2024-11-14T14:52:11,289 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1731595929525.cf7ad4aad0d0e3c5a9f31c16500a8e87.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34445/user/jenkins/test-data/35161b07-018a-0344-8c43-1519f3812567/data/default/testReplayEditsAfterRegionMovedWithMultiCF/cf7ad4aad0d0e3c5a9f31c16500a8e87/cf2/7be00c9408c842239946f9a0bad594bd] to archive 2024-11-14T14:52:11,290 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1731595929525.cf7ad4aad0d0e3c5a9f31c16500a8e87.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-14T14:52:11,292 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1731595929525.cf7ad4aad0d0e3c5a9f31c16500a8e87.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/35161b07-018a-0344-8c43-1519f3812567/data/default/testReplayEditsAfterRegionMovedWithMultiCF/cf7ad4aad0d0e3c5a9f31c16500a8e87/cf2/7be00c9408c842239946f9a0bad594bd to hdfs://localhost:34445/user/jenkins/test-data/35161b07-018a-0344-8c43-1519f3812567/archive/data/default/testReplayEditsAfterRegionMovedWithMultiCF/cf7ad4aad0d0e3c5a9f31c16500a8e87/cf2/7be00c9408c842239946f9a0bad594bd 2024-11-14T14:52:11,297 DEBUG [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34445/user/jenkins/test-data/35161b07-018a-0344-8c43-1519f3812567/data/default/testReplayEditsAfterRegionMovedWithMultiCF/cf7ad4aad0d0e3c5a9f31c16500a8e87/recovered.edits/17.seqid, newMaxSeqId=17, maxSeqId=8 2024-11-14T14:52:11,297 INFO [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1731595929525.cf7ad4aad0d0e3c5a9f31c16500a8e87. 2024-11-14T14:52:11,297 DEBUG [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1676): Region close journal for cf7ad4aad0d0e3c5a9f31c16500a8e87: Waiting for close lock at 1731595931266Running coprocessor pre-close hooks at 1731595931266Disabling compacts and flushes for region at 1731595931266Disabling writes for close at 1731595931267 (+1 ms)Writing region close event to WAL at 1731595931293 (+26 ms)Running coprocessor post-close hooks at 1731595931297 (+4 ms)Closed at 1731595931297 2024-11-14T14:52:11,297 INFO [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegionServer(3302): Adding cf7ad4aad0d0e3c5a9f31c16500a8e87 move to da1c6afcd1f9,34155,1731595913533 record at close sequenceid=12 2024-11-14T14:52:11,300 INFO [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] handler.UnassignRegionHandler(157): Closed cf7ad4aad0d0e3c5a9f31c16500a8e87 2024-11-14T14:52:11,301 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=cf7ad4aad0d0e3c5a9f31c16500a8e87, regionState=CLOSED 2024-11-14T14:52:11,306 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=10, state=RUNNABLE, hasLock=false; CloseRegionProcedure cf7ad4aad0d0e3c5a9f31c16500a8e87, server=da1c6afcd1f9,37695,1731595913427 because future has completed 2024-11-14T14:52:11,311 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=11, resume processing ppid=10 2024-11-14T14:52:11,311 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=10, state=SUCCESS, hasLock=false; CloseRegionProcedure cf7ad4aad0d0e3c5a9f31c16500a8e87, server=da1c6afcd1f9,37695,1731595913427 in 196 msec 2024-11-14T14:52:11,312 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=cf7ad4aad0d0e3c5a9f31c16500a8e87, REOPEN/MOVE; state=CLOSED, location=da1c6afcd1f9,34155,1731595913533; forceNewPlan=false, retain=false 2024-11-14T14:52:11,462 INFO [da1c6afcd1f9:42151 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-14T14:52:11,463 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=cf7ad4aad0d0e3c5a9f31c16500a8e87, regionState=OPENING, regionLocation=da1c6afcd1f9,34155,1731595913533 2024-11-14T14:52:11,466 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=cf7ad4aad0d0e3c5a9f31c16500a8e87, REOPEN/MOVE because future has completed 2024-11-14T14:52:11,466 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure cf7ad4aad0d0e3c5a9f31c16500a8e87, server=da1c6afcd1f9,34155,1731595913533}] 2024-11-14T14:52:11,622 INFO [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1731595929525.cf7ad4aad0d0e3c5a9f31c16500a8e87. 2024-11-14T14:52:11,622 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7752): Opening region: {ENCODED => cf7ad4aad0d0e3c5a9f31c16500a8e87, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1731595929525.cf7ad4aad0d0e3c5a9f31c16500a8e87.', STARTKEY => '', ENDKEY => ''} 2024-11-14T14:52:11,623 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF cf7ad4aad0d0e3c5a9f31c16500a8e87 2024-11-14T14:52:11,623 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1731595929525.cf7ad4aad0d0e3c5a9f31c16500a8e87.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T14:52:11,623 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7794): checking encryption for cf7ad4aad0d0e3c5a9f31c16500a8e87 2024-11-14T14:52:11,623 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7797): checking classloading for cf7ad4aad0d0e3c5a9f31c16500a8e87 2024-11-14T14:52:11,624 INFO [StoreOpener-cf7ad4aad0d0e3c5a9f31c16500a8e87-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region cf7ad4aad0d0e3c5a9f31c16500a8e87 2024-11-14T14:52:11,625 INFO [StoreOpener-cf7ad4aad0d0e3c5a9f31c16500a8e87-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cf7ad4aad0d0e3c5a9f31c16500a8e87 columnFamilyName cf1 2024-11-14T14:52:11,625 DEBUG [StoreOpener-cf7ad4aad0d0e3c5a9f31c16500a8e87-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T14:52:11,631 DEBUG [StoreOpener-cf7ad4aad0d0e3c5a9f31c16500a8e87-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:34445/user/jenkins/test-data/35161b07-018a-0344-8c43-1519f3812567/data/default/testReplayEditsAfterRegionMovedWithMultiCF/cf7ad4aad0d0e3c5a9f31c16500a8e87/cf1/efdb7fed42594e78b882b0ad2e13a8c1 2024-11-14T14:52:11,632 INFO [StoreOpener-cf7ad4aad0d0e3c5a9f31c16500a8e87-1 {}] regionserver.HStore(327): Store=cf7ad4aad0d0e3c5a9f31c16500a8e87/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T14:52:11,632 INFO [StoreOpener-cf7ad4aad0d0e3c5a9f31c16500a8e87-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region cf7ad4aad0d0e3c5a9f31c16500a8e87 2024-11-14T14:52:11,633 INFO [StoreOpener-cf7ad4aad0d0e3c5a9f31c16500a8e87-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cf7ad4aad0d0e3c5a9f31c16500a8e87 columnFamilyName cf2 2024-11-14T14:52:11,633 DEBUG [StoreOpener-cf7ad4aad0d0e3c5a9f31c16500a8e87-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T14:52:11,639 DEBUG [StoreOpener-cf7ad4aad0d0e3c5a9f31c16500a8e87-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:34445/user/jenkins/test-data/35161b07-018a-0344-8c43-1519f3812567/data/default/testReplayEditsAfterRegionMovedWithMultiCF/cf7ad4aad0d0e3c5a9f31c16500a8e87/cf2/2b8c1aa97dce4f2aa88a5be7c980475c 2024-11-14T14:52:11,639 INFO [StoreOpener-cf7ad4aad0d0e3c5a9f31c16500a8e87-1 {}] regionserver.HStore(327): Store=cf7ad4aad0d0e3c5a9f31c16500a8e87/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T14:52:11,639 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1038): replaying wal for cf7ad4aad0d0e3c5a9f31c16500a8e87 2024-11-14T14:52:11,640 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34445/user/jenkins/test-data/35161b07-018a-0344-8c43-1519f3812567/data/default/testReplayEditsAfterRegionMovedWithMultiCF/cf7ad4aad0d0e3c5a9f31c16500a8e87 2024-11-14T14:52:11,641 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34445/user/jenkins/test-data/35161b07-018a-0344-8c43-1519f3812567/data/default/testReplayEditsAfterRegionMovedWithMultiCF/cf7ad4aad0d0e3c5a9f31c16500a8e87 2024-11-14T14:52:11,641 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1048): stopping wal replay for cf7ad4aad0d0e3c5a9f31c16500a8e87 2024-11-14T14:52:11,642 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1060): Cleaning up temporary data for cf7ad4aad0d0e3c5a9f31c16500a8e87 2024-11-14T14:52:11,642 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-11-14T14:52:11,644 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1093): writing seq id for cf7ad4aad0d0e3c5a9f31c16500a8e87 2024-11-14T14:52:11,645 INFO [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1114): Opened cf7ad4aad0d0e3c5a9f31c16500a8e87; next sequenceid=18; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70692782, jitterRate=0.053404539823532104}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-11-14T14:52:11,645 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1122): Running coprocessor post-open hooks for cf7ad4aad0d0e3c5a9f31c16500a8e87 2024-11-14T14:52:11,646 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1006): Region open journal for cf7ad4aad0d0e3c5a9f31c16500a8e87: Running coprocessor pre-open hook at 1731595931623Writing region info on filesystem at 1731595931623Initializing all the Stores at 1731595931624 (+1 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731595931624Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731595931624Cleaning up temporary data from old regions at 1731595931642 (+18 ms)Running coprocessor post-open hooks at 1731595931645 (+3 ms)Region opened successfully at 1731595931646 (+1 ms) 2024-11-14T14:52:11,647 INFO [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1731595929525.cf7ad4aad0d0e3c5a9f31c16500a8e87., pid=12, masterSystemTime=1731595931619 2024-11-14T14:52:11,649 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1731595929525.cf7ad4aad0d0e3c5a9f31c16500a8e87. 2024-11-14T14:52:11,649 INFO [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1731595929525.cf7ad4aad0d0e3c5a9f31c16500a8e87. 2024-11-14T14:52:11,650 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=cf7ad4aad0d0e3c5a9f31c16500a8e87, regionState=OPEN, openSeqNum=18, regionLocation=da1c6afcd1f9,34155,1731595913533 2024-11-14T14:52:11,653 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=12, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure cf7ad4aad0d0e3c5a9f31c16500a8e87, server=da1c6afcd1f9,34155,1731595913533 because future has completed 2024-11-14T14:52:11,656 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=10 2024-11-14T14:52:11,656 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=10, state=SUCCESS, hasLock=false; OpenRegionProcedure cf7ad4aad0d0e3c5a9f31c16500a8e87, server=da1c6afcd1f9,34155,1731595913533 in 188 msec 2024-11-14T14:52:11,658 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=cf7ad4aad0d0e3c5a9f31c16500a8e87, REOPEN/MOVE in 550 msec 2024-11-14T14:52:11,659 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-14T14:52:11,661 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36392, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-14T14:52:11,663 ERROR [Time-limited test {}] regionserver.HRegionServer(2442): ***** ABORTING region server da1c6afcd1f9,34155,1731595913533: testing ***** 2024-11-14T14:52:11,663 ERROR [Time-limited test {}] regionserver.HRegionServer(2447): RegionServer abort: loaded coprocessors are: [org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint] 2024-11-14T14:52:11,665 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for java.lang:type=Memory 2024-11-14T14:52:11,666 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=IPC 2024-11-14T14:52:11,670 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Replication 2024-11-14T14:52:11,671 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Server 2024-11-14T14:52:11,680 INFO [Time-limited test {}] regionserver.HRegionServer(2451): Dump of metrics as JSON on abort: { "beans": [ { "name": "java.lang:type=Memory", "modelerType": "sun.management.MemoryImpl", "ObjectPendingFinalizationCount": 0, "HeapMemoryUsage": { "committed": 1048576000, "init": 1048576000, "max": 2306867200, "used": 510595768 }, "NonHeapMemoryUsage": { "committed": 172359680, "init": 7667712, "max": -1, "used": 169786440 }, "Verbose": false, "ObjectName": "java.lang:type=Memory" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=IPC", "modelerType": "RegionServer,sub=IPC", "tag.Context": "regionserver", "tag.Hostname": "da1c6afcd1f9", "queueSize": 0, "numCallsInGeneralQueue": 0, "numCallsInReplicationQueue": 0, "numCallsInBulkLoadQueue": 0, "numCallsInPriorityQueue": 0, "numCallsInMetaPriorityQueue": 0, "numOpenConnections": 3, "numActiveHandler": 0, "numActiveGeneralHandler": 0, "numActivePriorityHandler": 0, "numActiveReplicationHandler": 0, "numGeneralCallsDropped": 0, "numLifoModeSwitches": 0, "numCallsInWriteQueue": 0, "numActiveBulkLoadHandler": 0, "numCallsInReadQueue": 0, "numCallsInScanQueue": 0, "numActiveWriteHandler": 0, "numActiveReadHandler": 0, "numActiveScanHandler": 0, "nettyDirectMemoryUsage": 67108864, "nettyTotalPendingOutboundBytes": 0, "nettyMaxPendingOutboundBytes": 0, "receivedBytes": 11067, "exceptions.RegionMovedException": 1, "authenticationSuccesses": 0, "authorizationFailures": 0, "exceptions.requestTooBig": 0, "UnwritableTime_num_ops": 0, "UnwritableTime_min": 0, "UnwritableTime_max": 0, "UnwritableTime_mean": 0, "UnwritableTime_25th_percentile": 0, "UnwritableTime_median": 0, "UnwritableTime_75th_percentile": 0, "UnwritableTime_90th_percentile": 0, "UnwritableTime_95th_percentile": 0, "UnwritableTime_98th_percentile": 0, "UnwritableTime_99th_percentile": 0, "UnwritableTime_99.9th_percentile": 0, "exceptions.OutOfOrderScannerNextException": 0, "exceptions.rpcThrottling": 0, "exceptions.otherExceptions": 0, "ProcessCallTime_num_ops": 39, "ProcessCallTime_min": 0, "ProcessCallTime_max": 36, "ProcessCallTime_mean": 3, "ProcessCallTime_25th_percentile": 1, "ProcessCallTime_median": 2, "ProcessCallTime_75th_percentile": 3, "ProcessCallTime_90th_percentile": 5, "ProcessCallTime_95th_percentile": 12, "ProcessCallTime_98th_percentile": 36, "ProcessCallTime_99th_percentile": 36, "ProcessCallTime_99.9th_percentile": 36, "ProcessCallTime_TimeRangeCount_0-1": 14, "ProcessCallTime_TimeRangeCount_1-3": 12, "ProcessCallTime_TimeRangeCount_3-10": 3, "ProcessCallTime_TimeRangeCount_10-30": 1, "ProcessCallTime_TimeRangeCount_30-100": 1, "exceptions.callQueueTooBig": 0, "QueueCallTime_num_ops": 39, "QueueCallTime_min": 0, "QueueCallTime_max": 1, "QueueCallTime_mean": 0, "QueueCallTime_25th_percentile": 0, "QueueCallTime_median": 0, "QueueCallTime_75th_percentile": 0, "QueueCallTime_90th_percentile": 1, "QueueCallTime_95th_percentile": 1, "QueueCallTime_98th_percentile": 1, "QueueCallTime_99th_percentile": 1, "QueueCallTime_99.9th_percentile": 1, "QueueCallTime_TimeRangeCount_0-1": 31, "authenticationFailures": 0, "exceptions.multiResponseTooLarge": 0, "exceptions.callDropped": 0, "TotalCallTime_num_ops": 39, "TotalCallTime_min": 0, "TotalCallTime_max": 37, "TotalCallTime_mean": 3, "TotalCallTime_25th_percentile": 1, "TotalCallTime_median": 2, "TotalCallTime_75th_percentile": 3, "TotalCallTime_90th_percentile": 5, "TotalCallTime_95th_percentile": 13, "TotalCallTime_98th_percentile": 37, "TotalCallTime_99th_percentile": 37, "TotalCallTime_99.9th_percentile": 37, "TotalCallTime_TimeRangeCount_0-1": 13, "TotalCallTime_TimeRangeCount_1-3": 13, "TotalCallTime_TimeRangeCount_3-10": 3, "TotalCallTime_TimeRangeCount_10-30": 1, "TotalCallTime_TimeRangeCount_30-100": 1, "exceptions.RegionTooBusyException": 0, "exceptions.FailedSanityCheckException": 0, "ResponseSize_num_ops": 39, "ResponseSize_min": 0, "ResponseSize_max": 1138, "ResponseSize_mean": 158, "ResponseSize_25th_percentile": 2, "ResponseSize_median": 2, "ResponseSize_75th_percentile": 74, "ResponseSize_90th_percentile": 453, "ResponseSize_95th_percentile": 795, "ResponseSize_98th_percentile": 1001, "ResponseSize_99th_percentile": 1069, "ResponseSize_99.9th_percentile": 1131, "ResponseSize_SizeRangeCount_0-10": 22, "ResponseSize_SizeRangeCount_10-100": 5, "ResponseSize_SizeRangeCount_100-1000": 4, "exceptions.UnknownScannerException": 0, "exceptions": 1, "maxOutboundBytesExceeded": 0, "authenticationFallbacks": 0, "exceptions.quotaExceeded": 0, "exceptions.callTimedOut": 0, "exceptions.NotServingRegionException": 0, "authorizationSuccesses": 0, "exceptions.ScannerResetException": 0, "RequestSize_num_ops": 39, "RequestSize_min": 31, "RequestSize_max": 390, "RequestSize_mean": 204, "RequestSize_25th_percentile": 119, "RequestSize_median": 179, "RequestSize_75th_percentile": 330, "RequestSize_90th_percentile": 330, "RequestSize_95th_percentile": 365, "RequestSize_98th_percentile": 380, "RequestSize_99th_percentile": 385, "RequestSize_99.9th_percentile": 389, "RequestSize_SizeRangeCount_0-10": 2, "RequestSize_SizeRangeCount_100-1000": 29, "sentBytes": 5745 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Replication", "modelerType": "RegionServer,sub=Replication", "tag.Context": "regionserver", "tag.Hostname": "da1c6afcd1f9", "source.shippedHFiles": 0, "Source.ageOfLastShippedOp_num_ops": 0, "Source.ageOfLastShippedOp_min": 0, "Source.ageOfLastShippedOp_max": 0, "Source.ageOfLastShippedOp_mean": 0, "Source.ageOfLastShippedOp_25th_percentile": 0, "Source.ageOfLastShippedOp_median": 0, "Source.ageOfLastShippedOp_75th_percentile": 0, "Source.ageOfLastShippedOp_90th_percentile": 0, "Source.ageOfLastShippedOp_95th_percentile": 0, "Source.ageOfLastShippedOp_98th_percentile": 0, "Source.ageOfLastShippedOp_99th_percentile": 0, "Source.ageOfLastShippedOp_99.9th_percentile": 0, "source.uncleanlyClosedLogs": 0, "source.closedLogsWithUnknownFileLength": 0, "source.walReaderEditsBufferUsage": 0, "source.repeatedLogFileBytes": 0, "source.sizeOfHFileRefsQueue": 0, "source.logReadInBytes": 0, "source.completedRecoverQueues": 0, "source.sizeOfLogQueue": 0, "source.restartedLogReading": 0, "source.failedRecoverQueues": 0, "source.ignoredUncleanlyClosedLogContentsInBytes": 0, "Sink.ageOfLastAppliedOp_num_ops": 0, "Sink.ageOfLastAppliedOp_min": 0, "Sink.ageOfLastAppliedOp_max": 0, "Sink.ageOfLastAppliedOp_mean": 0, "Sink.ageOfLastAppliedOp_25th_percentile": 0, "Sink.ageOfLastAppliedOp_median": 0, "Sink.ageOfLastAppliedOp_75th_percentile": 0, "Sink.ageOfLastAppliedOp_90th_percentile": 0, "Sink.ageOfLastAppliedOp_95th_percentile": 0, "Sink.ageOfLastAppliedOp_98th_percentile": 0, "Sink.ageOfLastAppliedOp_99th_percentile": 0, "Sink.ageOfLastAppliedOp_99.9th_percentile": 0, "source.logEditsRead": 0, "source.numInitializing": 0, "source.shippedOps": 0, "sink.appliedHFiles": 0, "source.logEditsFiltered": 0, "source.shippedBytes": 0, "sink.appliedOps": 0, "source.completedLogs": 0, "source.failedBatches": 0, "sink.failedBatches": 0, "source.shippedBatches": 0, "sink.appliedBatches": 0 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Server", "modelerType": "RegionServer,sub=Server", "tag.zookeeperQuorum": "127.0.0.1:51231", "tag.serverName": "da1c6afcd1f9,46007,1731595913586", "tag.clusterId": "72d32361-5e69-4ee6-9ad5-db998b6b568a", "tag.Context": "regionserver", "tag.Hostname": "da1c6afcd1f9", "regionCount": 1, "storeCount": 4, "hlogFileCount": 6, "hlogFileSize": 0, "storeFileCount": 0, "maxStoreFileCount": 0, "memStoreSize": 952, "memStoreHeapSize": 2584, "memStoreOffHeapSize": 0, "storeFileSize": 0, "storeFileSizeGrowthRate": 0.0, "maxStoreFileAge": 0, "minStoreFileAge": 0, "avgStoreFileAge": 0, "numReferenceFiles": 0, "regionServerStartTime": 1731595913586, "averageRegionSize": 952, "storeFileIndexSize": 0, "staticIndexSize": 0, "staticBloomSize": 0, "bloomFilterRequestsCount": 0, "bloomFilterNegativeResultsCount": 0, "bloomFilterEligibleRequestsCount": 0, "mutationsWithoutWALCount": 0, "mutationsWithoutWALSize": 0, "percentFilesLocal": 0.0, "percentFilesLocalSecondaryRegions": 0.0, "totalBytesRead": 320685, "localBytesRead": 320685, "shortCircuitBytesRead": 0, "zeroCopyBytesRead": 0, "splitQueueLength": 0, "compactionQueueLength": 0, "smallCompactionQueueLength": 0, "largeCompactionQueueLength": 0, "flushQueueLength": 0, "blockCacheFreeSize": 922070024, "blockCacheCount": 0, "blockCacheDataBlockCount": 0, "blockCacheSize": 676856, "blockCacheCountHitPercent": 0.0, "blockCacheExpressHitPercent": 0.0, "l1CacheSize": 676856, "l1CacheFreeSize": 922070024, "l1CacheCount": 0, "l1CacheEvictionCount": 0, "l1CacheHitCount": 0, "l1CacheMissCount": 0, "l1CacheHitRatio": 0.0, "l1CacheMissRatio": 0.0, "l2CacheSize": 0, "l2CacheFreeSize": 0, "l2CacheCount": 0, "l2CacheEvictionCount": 0, "l2CacheHitCount": 0, "l2CacheMissCount": 0, "l2CacheHitRatio": 0.0, "l2CacheMissRatio": 0.0, "mobFileCacheCount": 0, "mobFileCacheHitPercent": 0.0, "readRequestRatePerSecond": 0.0, "writeRequestRatePerSecond": 0.6, "ByteBuffAllocatorHeapAllocationBytes": 227243, "ByteBuffAllocatorPoolAllocationBytes": 532480, "ByteBuffAllocatorHeapAllocationRatio": 0.4006422205053119, "ByteBuffAllocatorTotalBufferCount": 186, "ByteBuffAllocatorUsedBufferCount": 1, "activeScanners": 0, "totalRequestCount": 25, "totalRowActionRequestCount": 12, "readRequestCount": 7, "cpRequestCount": 0, "filteredReadRequestCount": 0, "writeRequestCount": 5, "rpcGetRequestCount": 2, "rpcFullScanRequestCount": 0, "rpcScanRequestCount": 8, "rpcMultiRequestCount": 0, "rpcMutateRequestCount": 15, "checkMutateFailedCount": 0, "checkMutatePassedCount": 0, "blockCacheHitCount": 0, "blockCacheHitCountPrimary": 0, "blockCacheHitCachingCount": 0, "blockCacheMissCount": 0, "blockCacheMissCountPrimary": 0, "blockCacheMissCachingCount": 0, "blockCacheEvictionCount": 0, "blockCacheEvictionCountPrimary": 0, "blockCacheFailedInsertionCount": 0, "blockCacheDataMissCount": 0, "blockCacheLeafIndexMissCount": 0, "blockCacheBloomChunkMissCount": 0, "blockCacheMetaMissCount": 0, "blockCacheRootIndexMissCount": 0, "blockCacheIntermediateIndexMissCount": 0, "blockCacheFileInfoMissCount": 0, "blockCacheGeneralBloomMetaMissCount": 0, "blockCacheDeleteFamilyBloomMissCount": 0, "blockCacheTrailerMissCount": 0, "blockCacheDataHitCount": 0, "blockCacheLeafIndexHitCount": 0, "blockCacheBloomChunkHitCount": 0, "blockCacheMetaHitCount": 0, "blockCacheRootIndexHitCount": 0, "blockCacheIntermediateIndexHitCount": 0, "blockCacheFileInfoHitCount": 0, "blockCacheGeneralBloomMetaHitCount": 0, "blockCacheDeleteFamilyBloomHitCount": 0, "blockCacheTrailerHitCount": 0, "updatesBlockedTime": 0, "flushedCellsCount": 0, "compactedCellsCount": 0, "majorCompactedCellsCount": 0, "flushedCellsSize": 0, "compactedCellsSize": 0, "majorCompactedCellsSize": 0, "cellsCountCompactedFromMob": 0, "cellsCountCompactedToMob": 0, "cellsSizeCompactedFromMob": 0, "cellsSizeCompactedToMob": 0, "mobFlushCount": 0, "mobFlushedCellsCount": 0, "mobFlushedCellsSize": 0, "mobScanCellsCount": 0, "mobScanCellsSize": 0, "mobFileCacheAccessCount": 0, "mobFileCacheMissCount": 0, "mobFileCacheEvictedCount": 0, "hedgedReads": 0, "hedgedReadWins": 0, "hedgedReadOpsInCurThread": 0, "blockedRequestCount": 0, "CheckAndMutate_num_ops": 0, "CheckAndMutate_min": 0, "CheckAndMutate_max": 0, "CheckAndMutate_mean": 0, "CheckAndMutate_25th_percentile": 0, "CheckAndMutate_median": 0, "CheckAndMutate_75th_percentile": 0, "CheckAndMutate_90th_percentile": 0, "CheckAndMutate_95th_percentile": 0, "CheckAndMutate_98th_percentile": 0, "CheckAndMutate_99th_percentile": 0, "CheckAndMutate_99.9th_percentile": 0, "MajorCompactionTime_num_ops": 0, "MajorCompactionTime_min": 0, "MajorCompactionTime_max": 0, "MajorCompactionTime_mean": 0, "MajorCompactionTime_25th_percentile": 0, "MajorCompactionTime_median": 0, "MajorCompactionTime_75th_percentile": 0, "MajorCompactionTime_90th_percentile": 0, "MajorCompactionTime_95th_percentile": 0, "MajorCompactionTime_98th_percentile": 0, "MajorCompactionTime_99th_percentile": 0, "MajorCompactionTime_99.9th_percentile": 0, "ScanTime_num_ops": 8, "ScanTime_min": 1, "ScanTime_max": 2, "ScanTime_mean": 1, "ScanTime_25th_percentile": 1, "ScanTime_median": 1, "ScanTime_75th_percentile": 2, "ScanTime_90th_percentile": 2, "ScanTime_95th_percentile": 2, "ScanTime_98th_percentile": 2, "ScanTime_99th_percentile": 2, "ScanTime_99.9th_percentile": 2, "ScanTime_TimeRangeCount_0-1": 3, "ScanTime_TimeRangeCount_1-3": 1, "CheckAndMutateBlockBytesScanned_num_ops": 0, "CheckAndMutateBlockBytesScanned_min": 0, "CheckAndMutateBlockBytesScanned_max": 0, "CheckAndMutateBlockBytesScanned_mean": 0, "CheckAndMutateBlockBytesScanned_25th_percentile": 0, "CheckAndMutateBlockBytesScanned_median": 0, "CheckAndMutateBlockBytesScanned_75th_percentile": 0, "CheckAndMutateBlockBytesScanned_90th_percentile": 0, "CheckAndMutateBlockBytesScanned_95th_percentile": 0, "CheckAndMutateBlockBytesScanned_98th_percentile": 0, "CheckAndMutateBlockBytesScanned_99th_percentile": 0, "CheckAndMutateBlockBytesScanned_99.9th_percentile": 0, "Put_num_ops": 15, "Put_min": 1, "Put_max": 3, "Put_mean": 1, "Put_25th_percentile": 1, "Put_median": 2, "Put_75th_percentile": 2, "Put_90th_percentile": 2, "Put_95th_percentile": 3, "Put_98th_percentile": 3, "Put_99th_percentile": 3, "Put_99.9th_percentile": 3, "Put_TimeRangeCount_0-1": 6, "Put_TimeRangeCount_1-3": 7, "splitRequestCount": 0, "AppendBlockBytesScanned_num_ops": 0, "AppendBlockBytesScanned_min": 0, "AppendBlockBytesScanned_max": 0, "AppendBlockBytesScanned_mean": 0, "AppendBlockBytesScanned_25th_percentile": 0, "AppendBlockBytesScanned_median": 0, "AppendBlockBytesScanned_75th_percentile": 0, "AppendBlockBytesScanned_90th_percentile": 0, "AppendBlockBytesScanned_95th_percentile": 0, "AppendBlockBytesScanned_98th_percentile": 0, "AppendBlockBytesScanned_99th_percentile": 0, "AppendBlockBytesScanned_99.9th_percentile": 0, "PutBatch_num_ops": 0, "PutBatch_min": 0, "PutBatch_max": 0, "PutBatch_mean": 0, "PutBatch_25th_percentile": 0, "PutBatch_median": 0, "PutBatch_75th_percentile": 0, "PutBatch_90th_percentile": 0, "PutBatch_95th_percentile": 0, "PutBatch_98th_percentile": 0, "PutBatch_99th_percentile": 0, "PutBatch_99.9th_percentile": 0, "IncrementBlockBytesScanned_num_ops": 0, "IncrementBlockBytesScanned_min": 0, "IncrementBlockBytesScanned_max": 0, "IncrementBlockBytesScanned_mean": 0, "IncrementBlockBytesScanned_25th_percentile": 0, "IncrementBlockBytesScanned_median": 0, "IncrementBlockBytesScanned_75th_percentile": 0, "IncrementBlockBytesScanned_90th_percentile": 0, "IncrementBlockBytesScanned_95th_percentile": 0, "IncrementBlockBytesScanned_98th_percentile": 0, "IncrementBlockBytesScanned_99th_percentile": 0, "IncrementBlockBytesScanned_99.9th_percentile": 0, "SplitTime_num_ops": 0, "SplitTime_min": 0, "SplitTime_max": 0, "SplitTime_mean": 0, "SplitTime_25th_percentile": 0, "SplitTime_median": 0, "SplitTime_75th_percentile": 0, "SplitTime_90th_percentile": 0, "SplitTime_95th_percentile": 0, "SplitTime_98th_percentile": 0, "SplitTime_99th_percentile": 0, "SplitTime_99.9th_percentile": 0, "GetBlockBytesScanned_num_ops": 0, "GetBlockBytesScanned_min": 0, "GetBlockBytesScanned_max": 0, "GetBlockBytesScanned_mean": 0, "GetBlockBytesScanned_25th_percentile": 0, "GetBlockBytesScanned_median": 0, "GetBlockBytesScanned_75th_percentile": 0, "GetBlockBytesScanned_90th_percentile": 0, "GetBlockBytesScanned_95th_percentile": 0, "GetBlockBytesScanned_98th_percentile": 0, "GetBlockBytesScanned_99th_percentile": 0, "GetBlockBytesScanned_99.9th_percentile": 0, "majorCompactedInputBytes": 0, "slowAppendCount": 0, "flushedOutputBytes": 0, "Replay_num_ops": 0, "Replay_min": 0, "Replay_max": 0, "Replay_mean": 0, "Replay_25th_percentile": 0, "Replay_median": 0, "Replay_75th_percentile": 0, "Replay_90th_percentile": 0, "Replay_95th_percentile": 0, "Replay_98th_percentile": 0, "Replay_99th_percentile": 0, "Replay_99.9th_percentile": 0, "MajorCompactionInputSize_num_ops": 0, "MajorCompactionInputSize_min": 0, "MajorCompactionInputSize_max": 0, "MajorCompactionInputSize_mean": 0, "MajorCompactionInputSize_25th_percentile": 0, "MajorCompactionInputSize_median": 0, "MajorCompactionInputSize_75th_percentile": 0, "MajorCompactionInputSize_90th_percentile": 0, "MajorCompactionInputSize_95th_percentile": 0, "MajorCompactionInputSize_98th_percentile": 0, "MajorCompactionInputSize_99th_percentile": 0, "MajorCompactionInputSize_99.9th_percentile": 0, "pauseInfoThresholdExceeded": 0, "CheckAndDelete_num_ops": 0, "CheckAndDelete_min": 0, "CheckAndDelete_max": 0, "CheckAndDelete_mean": 0, "CheckAndDelete_25th_percentile": 0, "CheckAndDelete_median": 0, "CheckAndDelete_75th_percentile": 0, "CheckAndDelete_90th_percentile": 0, "CheckAndDelete_95th_percentile": 0, "CheckAndDelete_98th_percentile": 0, "CheckAndDelete_99th_percentile": 0, "CheckAndDelete_99.9th_percentile": 0, "CompactionInputSize_num_ops": 0, "CompactionInputSize_min": 0, "CompactionInputSize_max": 0, "CompactionInputSize_mean": 0, "CompactionInputSize_25th_percentile": 0, "CompactionInputSize_median": 0, "CompactionInputSize_75th_percentile": 0, "CompactionInputSize_90th_percentile": 0, "CompactionInputSize_95th_percentile": 0, "CompactionInputSize_98th_percentile": 0, "CompactionInputSize_99th_percentile": 0, "CompactionInputSize_99.9th_percentile": 0, "flushedMemstoreBytes": 0, "majorCompactedOutputBytes": 0, "slowPutCount": 0, "compactedInputBytes": 0, "FlushOutputSize_num_ops": 0, "FlushOutputSize_min": 0, "FlushOutputSize_max": 0, "FlushOutputSize_mean": 0, "FlushOutputSize_25th_percentile": 0, "FlushOutputSize_median": 0, "FlushOutputSize_75th_percentile": 0, "FlushOutputSize_90th_percentile": 0, "FlushOutputSize_95th_percentile": 0, "FlushOutputSize_98th_percentile": 0, "FlushOutputSize_99th_percentile": 0, "FlushOutputSize_99.9th_percentile": 0, "PauseTimeWithGc_num_ops": 0, "PauseTimeWithGc_min": 0, "PauseTimeWithGc_max": 0, "PauseTimeWithGc_mean": 0, "PauseTimeWithGc_25th_percentile": 0, "PauseTimeWithGc_median": 0, "PauseTimeWithGc_75th_percentile": 0, "PauseTimeWithGc_90th_percentile": 0, "PauseTimeWithGc_95th_percentile": 0, "PauseTimeWithGc_98th_percentile": 0, "PauseTimeWithGc_99th_percentile": 0, "PauseTimeWithGc_99.9th_percentile": 0, "compactedOutputBytes": 0, "pauseWarnThresholdExceeded": 0, "ScanBlockBytesScanned_num_ops": 0, "ScanBlockBytesScanned_min": 0, "ScanBlockBytesScanned_max": 0, "ScanBlockBytesScanned_mean": 0, "ScanBlockBytesScanned_25th_percentile": 0, "ScanBlockBytesScanned_median": 0, "ScanBlockBytesScanned_75th_percentile": 0, "ScanBlockBytesScanned_90th_percentile": 0, "ScanBlockBytesScanned_95th_percentile": 0, "ScanBlockBytesScanned_98th_percentile": 0, "ScanBlockBytesScanned_99th_percentile": 0, "ScanBlockBytesScanned_99.9th_percentile": 0, "Increment_num_ops": 0, "Increment_min": 0, "Increment_max": 0, "Increment_mean": 0, "Increment_25th_percentile": 0, "Increment_median": 0, "Increment_75th_percentile": 0, "Increment_90th_percentile": 0, "Increment_95th_percentile": 0, "Increment_98th_percentile": 0, "Increment_99th_percentile": 0, "Increment_99.9th_percentile": 0, "Delete_num_ops": 0, "Delete_min": 0, "Delete_max": 0, "Delete_mean": 0, "Delete_25th_percentile": 0, "Delete_median": 0, "Delete_75th_percentile": 0, "Delete_90th_percentile": 0, "Delete_95th_percentile": 0, "Delete_98th_percentile": 0, "Delete_99th_percentile": 0, "Delete_99.9th_percentile": 0, "DeleteBatch_num_ops": 0, "DeleteBatch_min": 0, "DeleteBatch_max": 0, "DeleteBatch_mean": 0, "DeleteBatch_25th_percentile": 0, "DeleteBatch_median": 0, "DeleteBatch_75th_percentile": 0, "DeleteBatch_90th_percentile": 0, "DeleteBatch_95th_percentile": 0, "DeleteBatch_98th_percentile": 0, "DeleteBatch_99th_percentile": 0, "DeleteBatch_99.9th_percentile": 0, "blockBytesScannedCount": 0, "FlushMemstoreSize_num_ops": 0, "FlushMemstoreSize_min": 0, "FlushMemstoreSize_max": 0, "FlushMemstoreSize_mean": 0, "FlushMemstoreSize_25th_percentile": 0, "FlushMemstoreSize_median": 0, "FlushMemstoreSize_75th_percentile": 0, "FlushMemstoreSize_90th_percentile": 0, "FlushMemstoreSize_95th_percentile": 0, "FlushMemstoreSize_98th_percentile": 0, "FlushMemstoreSize_99th_percentile": 0, "FlushMemstoreSize_99.9th_percentile": 0, "CompactionInputFileCount_num_ops": 0, "CompactionInputFileCount_min": 0, "CompactionInputFileCount_max": 0, "CompactionInputFileCount_mean": 0, "CompactionInputFileCount_25th_percentile": 0, "CompactionInputFileCount_median": 0, "CompactionInputFileCount_75th_percentile": 0, "CompactionInputFileCount_90th_percentile": 0, "CompactionInputFileCount_95th_percentile": 0, "CompactionInputFileCount_98th_percentile": 0, "CompactionInputFileCount_99th_percentile": 0, "CompactionInputFileCount_99.9th_percentile": 0, "CompactionTime_num_ops": 0, "CompactionTime_min": 0, "CompactionTime_max": 0, "CompactionTime_mean": 0, "CompactionTime_25th_percentile": 0, "CompactionTime_median": 0, "CompactionTime_75th_percentile": 0, "CompactionTime_90th_percentile": 0, "CompactionTime_95th_percentile": 0, "CompactionTime_98th_percentile": 0, "CompactionTime_99th_percentile": 0, "CompactionTime_99.9th_percentile": 0, "Get_num_ops": 2, "Get_min": 0, "Get_max": 0, "Get_mean": 0, "Get_25th_percentile": 0, "Get_median": 0, "Get_75th_percentile": 0, "Get_90th_percentile": 0, "Get_95th_percentile": 0, "Get_98th_percentile": 0, "Get_99th_percentile": 0, "Get_99.9th_percentile": 0, "Get_TimeRangeCount_0-1": 1, "MajorCompactionInputFileCount_num_ops": 0, "MajorCompactionInputFileCount_min": 0, "MajorCompactionInputFileCount_max": 0, "MajorCompactionInputFileCount_mean": 0, "MajorCompactionInputFileCount_25th_percentile": 0, "MajorCompactionInputFileCount_median": 0, "MajorCompactionInputFileCount_75th_percentile": 0, "MajorCompactionInputFileCount_90th_percentile": 0, "MajorCompactionInputFileCount_95th_percentile": 0, "MajorCompactionInputFileCount_98th_percentile": 0, "MajorCompactionInputFileCount_99th_percentile": 0, "MajorCompactionInputFileCount_99.9th_percentile": 0, "scannerLeaseExpiredCount": 0, "CheckAndPut_num_ops": 0, "CheckAndPut_min": 0, "CheckAndPut_max": 0, "CheckAndPut_mean": 0, "CheckAndPut_25th_percentile": 0, "CheckAndPut_median": 0, "CheckAndPut_75th_percentile": 0, "CheckAndPut_90th_percentile": 0, "CheckAndPut_95th_percentile": 0, "CheckAndPut_98th_percentile": 0, "CheckAndPut_99th_percentile": 0, "CheckAndPut_99.9th_percentile": 0, "MajorCompactionOutputSize_num_ops": 0, "MajorCompactionOutputSize_min": 0, "MajorCompactionOutputSize_max": 0, "MajorCompactionOutputSize_mean": 0, "MajorCompactionOutputSize_25th_percentile": 0, "MajorCompactionOutputSize_median": 0, "MajorCompactionOutputSize_75th_percentile": 0, "MajorCompactionOutputSize_90th_percentile": 0, "MajorCompactionOutputSize_95th_percentile": 0, "MajorCompactionOutputSize_98th_percentile": 0, "MajorCompactionOutputSize_99th_percentile": 0, "MajorCompactionOutputSize_99.9th_percentile": 0, "CompactionOutputFileCount_num_ops": 0, "CompactionOutputFileCount_min": 0, "CompactionOutputFileCount_max": 0, "CompactionOutputFileCount_mean": 0, "CompactionOutputFileCount_25th_percentile": 0, "CompactionOutputFileCount_median": 0, "CompactionOutputFileCount_75th_percentile": 0, "CompactionOutputFileCount_90th_percentile": 0, "CompactionOutputFileCount_95th_percentile": 0, "CompactionOutputFileCount_98th_percentile": 0, "CompactionOutputFileCount_99th_percentile": 0, "CompactionOutputFileCount_99.9th_percentile": 0, "slowDeleteCount": 0, "FlushTime_num_ops": 0, "FlushTime_min": 0, "FlushTime_max": 0, "FlushTime_mean": 0, "FlushTime_25th_percentile": 0, "FlushTime_median": 0, "FlushTime_75th_percentile": 0, "FlushTime_90th_percentile": 0, "FlushTime_95th_percentile": 0, "FlushTime_98th_percentile": 0, "FlushTime_99th_percentile": 0, "FlushTime_99.9th_percentile": 0, "splitSuccessCount": 0, "MajorCompactionOutputFileCount_num_ops": 0, "MajorCompactionOutputFileCount_min": 0, "MajorCompactionOutputFileCount_max": 0, "MajorCompactionOutputFileCount_mean": 0, "MajorCompactionOutputFileCount_25th_percentile": 0, "MajorCompactionOutputFileCount_median": 0, "MajorCompactionOutputFileCount_75th_percentile": 0, "MajorCompactionOutputFileCount_90th_percentile": 0, "MajorCompactionOutputFileCount_95th_percentile": 0, "MajorCompactionOutputFileCount_98th_percentile": 0, "MajorCompactionOutputFileCount_99th_percentile": 0, "MajorCompactionOutputFileCount_99.9th_percentile": 0, "slowGetCount": 0, "ScanSize_num_ops": 8, "ScanSize_min": 1112, "ScanSize_max": 1112, "ScanSize_mean": 1112, "ScanSize_25th_percentile": 1112, "ScanSize_median": 1112, "ScanSize_75th_percentile": 1112, "ScanSize_90th_percentile": 1112, "ScanSize_95th_percentile": 1112, "ScanSize_98th_percentile": 1112, "ScanSize_99th_percentile": 1112, "ScanSize_99.9th_percentile": 1112, "ScanSize_SizeRangeCount_100-1000": 4, "CompactionOutputSize_num_ops": 0, "CompactionOutputSize_min": 0, "CompactionOutputSize_max": 0, "CompactionOutputSize_mean": 0, "CompactionOutputSize_25th_percentile": 0, "CompactionOutputSize_median": 0, "CompactionOutputSize_75th_percentile": 0, "CompactionOutputSize_90th_percentile": 0, "CompactionOutputSize_95th_percentile": 0, "CompactionOutputSize_98th_percentile": 0, "CompactionOutputSize_99th_percentile": 0, "CompactionOutputSize_99.9th_percentile": 0, "PauseTimeWithoutGc_num_ops": 0, "PauseTimeWithoutGc_min": 0, "PauseTimeWithoutGc_max": 0, "PauseTimeWithoutGc_mean": 0, "PauseTimeWithoutGc_25th_percentile": 0, "PauseTimeWithoutGc_median": 0, "PauseTimeWithoutGc_75th_percentile": 0, "PauseTimeWithoutGc_90th_percentile": 0, "PauseTimeWithoutGc_95th_percentile": 0, "PauseTimeWithoutGc_98th_percentile": 0, "PauseTimeWithoutGc_99th_percentile": 0, "PauseTimeWithoutGc_99.9th_percentile": 0, "slowIncrementCount": 0, "Append_num_ops": 0, "Append_min": 0, "Append_max": 0, "Append_mean": 0, "Append_25th_percentile": 0, "Append_median": 0, "Append_75th_percentile": 0, "Append_90th_percentile": 0, "Append_95th_percentile": 0, "Append_98th_percentile": 0, "Append_99th_percentile": 0, "Append_99.9th_percentile": 0, "Bulkload_count": 0, "Bulkload_mean_rate": 0.0, "Bulkload_1min_rate": 0.0, "Bulkload_5min_rate": 0.0, "Bulkload_15min_rate": 0.0, "Bulkload_num_ops": 0, "Bulkload_min": 0, "Bulkload_max": 0, "Bulkload_mean": 0, "Bulkload_25th_percentile": 0, "Bulkload_median": 0, "Bulkload_75th_percentile": 0, "Bulkload_90th_percentile": 0, "Bulkload_95th_percentile": 0, "Bulkload_98th_percentile": 0, "Bulkload_99th_percentile": 0, "Bulkload_99.9th_percentile": 0 } ] } 2024-11-14T14:52:11,683 WARN [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42151 {}] master.MasterRpcServices(700): da1c6afcd1f9,34155,1731595913533 reported a fatal error: ***** ABORTING region server da1c6afcd1f9,34155,1731595913533: testing ***** 2024-11-14T14:52:11,685 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'da1c6afcd1f9,34155,1731595913533' ***** 2024-11-14T14:52:11,685 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: testing 2024-11-14T14:52:11,685 INFO [RS:1;da1c6afcd1f9:34155 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-14T14:52:11,686 INFO [RS:1;da1c6afcd1f9:34155 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager abruptly. 2024-11-14T14:52:11,686 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-14T14:52:11,686 INFO [RS:1;da1c6afcd1f9:34155 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager abruptly. 2024-11-14T14:52:11,686 INFO [RS:1;da1c6afcd1f9:34155 {}] regionserver.HRegionServer(3091): Received CLOSE for cf7ad4aad0d0e3c5a9f31c16500a8e87 2024-11-14T14:52:11,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Get size: 140 connection: 172.17.0.2:43358 deadline: 1731595991686, exception=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=da1c6afcd1f9 port=34155 startCode=1731595913533. As of locationSeqNum=12. 2024-11-14T14:52:11,686 INFO [RS:1;da1c6afcd1f9:34155 {}] regionserver.HRegionServer(956): aborting server da1c6afcd1f9,34155,1731595913533 2024-11-14T14:52:11,687 INFO [RS:1;da1c6afcd1f9:34155 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-14T14:52:11,687 DEBUG [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing cf7ad4aad0d0e3c5a9f31c16500a8e87, disabling compactions & flushes 2024-11-14T14:52:11,687 INFO [RS:1;da1c6afcd1f9:34155 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;da1c6afcd1f9:34155. 2024-11-14T14:52:11,687 DEBUG [RPCClient-NioEventLoopGroup-6-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1731595929525.cf7ad4aad0d0e3c5a9f31c16500a8e87., hostname=da1c6afcd1f9,37695,1731595913427, seqNum=5 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1731595929525.cf7ad4aad0d0e3c5a9f31c16500a8e87., hostname=da1c6afcd1f9,37695,1731595913427, seqNum=5, error=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=da1c6afcd1f9 port=34155 startCode=1731595913533. As of locationSeqNum=12. 2024-11-14T14:52:11,687 INFO [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1731595929525.cf7ad4aad0d0e3c5a9f31c16500a8e87. 2024-11-14T14:52:11,687 DEBUG [RPCClient-NioEventLoopGroup-6-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1731595929525.cf7ad4aad0d0e3c5a9f31c16500a8e87., hostname=da1c6afcd1f9,37695,1731595913427, seqNum=5 is org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=da1c6afcd1f9 port=34155 startCode=1731595913533. As of locationSeqNum=12. 2024-11-14T14:52:11,687 DEBUG [RS:1;da1c6afcd1f9:34155 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-14T14:52:11,687 DEBUG [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731595929525.cf7ad4aad0d0e3c5a9f31c16500a8e87. 2024-11-14T14:52:11,687 DEBUG [RPCClient-NioEventLoopGroup-6-7 {}] client.AsyncRegionLocatorHelper(84): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1731595929525.cf7ad4aad0d0e3c5a9f31c16500a8e87., hostname=da1c6afcd1f9,37695,1731595913427, seqNum=5 with the new location region=testReplayEditsAfterRegionMovedWithMultiCF,,1731595929525.cf7ad4aad0d0e3c5a9f31c16500a8e87., hostname=da1c6afcd1f9,34155,1731595913533, seqNum=12 constructed by org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=da1c6afcd1f9 port=34155 startCode=1731595913533. As of locationSeqNum=12. 2024-11-14T14:52:11,687 DEBUG [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731595929525.cf7ad4aad0d0e3c5a9f31c16500a8e87. after waiting 0 ms 2024-11-14T14:52:11,687 DEBUG [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1731595929525.cf7ad4aad0d0e3c5a9f31c16500a8e87. 2024-11-14T14:52:11,688 DEBUG [RS:1;da1c6afcd1f9:34155 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T14:52:11,688 INFO [RS:1;da1c6afcd1f9:34155 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-14T14:52:11,688 DEBUG [RS:1;da1c6afcd1f9:34155 {}] regionserver.HRegionServer(1325): Online Regions={cf7ad4aad0d0e3c5a9f31c16500a8e87=testReplayEditsAfterRegionMovedWithMultiCF,,1731595929525.cf7ad4aad0d0e3c5a9f31c16500a8e87.} 2024-11-14T14:52:11,689 DEBUG [RS:1;da1c6afcd1f9:34155 {}] regionserver.HRegionServer(1351): Waiting on cf7ad4aad0d0e3c5a9f31c16500a8e87 2024-11-14T14:52:11,690 INFO [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1731595929525.cf7ad4aad0d0e3c5a9f31c16500a8e87. 2024-11-14T14:52:11,690 DEBUG [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for cf7ad4aad0d0e3c5a9f31c16500a8e87: Waiting for close lock at 1731595931687Running coprocessor pre-close hooks at 1731595931687Disabling compacts and flushes for region at 1731595931687Disabling writes for close at 1731595931687Writing region close event to WAL at 1731595931690 (+3 ms)Running coprocessor post-close hooks at 1731595931690Closed at 1731595931690 2024-11-14T14:52:11,691 DEBUG [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1731595929525.cf7ad4aad0d0e3c5a9f31c16500a8e87. 2024-11-14T14:52:11,769 INFO [regionserver/da1c6afcd1f9:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-14T14:52:11,795 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server da1c6afcd1f9,34155,1731595913533 aborting at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processRequest(ServerRpcConnection.java:564) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processOneRpc(ServerRpcConnection.java:364) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyServerRpcConnection.process(NettyServerRpcConnection.java:89) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:56) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:31) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:52:11,796 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1731595929525.cf7ad4aad0d0e3c5a9f31c16500a8e87., hostname=da1c6afcd1f9,34155,1731595913533, seqNum=12 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1731595929525.cf7ad4aad0d0e3c5a9f31c16500a8e87., hostname=da1c6afcd1f9,34155,1731595913533, seqNum=12, error=org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server da1c6afcd1f9,34155,1731595913533 aborting 2024-11-14T14:52:11,796 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1731595929525.cf7ad4aad0d0e3c5a9f31c16500a8e87., hostname=da1c6afcd1f9,34155,1731595913533, seqNum=12 is org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server da1c6afcd1f9,34155,1731595913533 aborting 2024-11-14T14:52:11,797 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testReplayEditsAfterRegionMovedWithMultiCF,,1731595929525.cf7ad4aad0d0e3c5a9f31c16500a8e87., hostname=da1c6afcd1f9,34155,1731595913533, seqNum=12 from cache 2024-11-14T14:52:11,889 INFO [RS:1;da1c6afcd1f9:34155 {}] regionserver.HRegionServer(976): stopping server da1c6afcd1f9,34155,1731595913533; all regions closed. 2024-11-14T14:52:11,892 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42819 is added to blk_1073741835_1011 (size=2288) 2024-11-14T14:52:11,893 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36189 is added to blk_1073741835_1011 (size=2288) 2024-11-14T14:52:11,893 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44549 is added to blk_1073741835_1011 (size=2288) 2024-11-14T14:52:11,893 DEBUG [RS:1;da1c6afcd1f9:34155 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T14:52:11,893 INFO [RS:1;da1c6afcd1f9:34155 {}] regionserver.LeaseManager(133): Closed leases 2024-11-14T14:52:11,894 INFO [RS:1;da1c6afcd1f9:34155 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-14T14:52:11,894 INFO [RS:1;da1c6afcd1f9:34155 {}] hbase.ChoreService(370): Chore service for: regionserver/da1c6afcd1f9:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-14T14:52:11,894 INFO [regionserver/da1c6afcd1f9:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-14T14:52:11,894 INFO [RS:1;da1c6afcd1f9:34155 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-14T14:52:11,894 INFO [RS:1;da1c6afcd1f9:34155 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-14T14:52:11,894 INFO [RS:1;da1c6afcd1f9:34155 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-14T14:52:11,894 INFO [RS:1;da1c6afcd1f9:34155 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-14T14:52:11,895 INFO [RS:1;da1c6afcd1f9:34155 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:34155 2024-11-14T14:52:11,901 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34155-0x101a74438830002, quorum=127.0.0.1:51231, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/da1c6afcd1f9,34155,1731595913533 2024-11-14T14:52:11,901 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42151-0x101a74438830000, quorum=127.0.0.1:51231, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-14T14:52:11,901 INFO [RS:1;da1c6afcd1f9:34155 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-14T14:52:11,903 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [da1c6afcd1f9,34155,1731595913533] 2024-11-14T14:52:11,904 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/da1c6afcd1f9,34155,1731595913533 already deleted, retry=false 2024-11-14T14:52:11,905 INFO [RegionServerTracker-0 {}] master.ServerManager(695): Processing expiration of da1c6afcd1f9,34155,1731595913533 on da1c6afcd1f9,42151,1731595912734 2024-11-14T14:52:11,910 DEBUG [RegionServerTracker-0 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure da1c6afcd1f9,34155,1731595913533, splitWal=true, meta=false 2024-11-14T14:52:11,912 INFO [RegionServerTracker-0 {}] assignment.AssignmentManager(1999): Scheduled ServerCrashProcedure pid=13 for da1c6afcd1f9,34155,1731595913533 (carryingMeta=false) da1c6afcd1f9,34155,1731595913533/CRASHED/regionCount=1/lock=java.util.concurrent.locks.ReentrantReadWriteLock@3b66e0fe[Write locks = 1, Read locks = 0], oldState=ONLINE. 2024-11-14T14:52:11,912 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(169): Start pid=13, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure da1c6afcd1f9,34155,1731595913533, splitWal=true, meta=false 2024-11-14T14:52:11,914 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(207): da1c6afcd1f9,34155,1731595913533 had 1 regions 2024-11-14T14:52:11,915 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=13, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure da1c6afcd1f9,34155,1731595913533, splitWal=true, meta=false, isMeta: false 2024-11-14T14:52:11,916 DEBUG [PEWorker-1 {}] master.MasterWalManager(329): Renamed region directory: hdfs://localhost:34445/user/jenkins/test-data/35161b07-018a-0344-8c43-1519f3812567/WALs/da1c6afcd1f9,34155,1731595913533-splitting 2024-11-14T14:52:11,917 INFO [PEWorker-1 {}] master.SplitWALManager(105): da1c6afcd1f9,34155,1731595913533 WAL count=1, meta=false 2024-11-14T14:52:11,920 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure da1c6afcd1f9%2C34155%2C1731595913533.1731595915018}] 2024-11-14T14:52:11,925 DEBUG [PEWorker-2 {}] master.SplitWALManager(158): Acquired split WAL worker=da1c6afcd1f9,46007,1731595913586 2024-11-14T14:52:11,927 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=15, ppid=14, state=RUNNABLE, hasLock=false; SplitWALRemoteProcedure da1c6afcd1f9%2C34155%2C1731595913533.1731595915018, worker=da1c6afcd1f9,46007,1731595913586}] 2024-11-14T14:52:12,004 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34155-0x101a74438830002, quorum=127.0.0.1:51231, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-14T14:52:12,004 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34155-0x101a74438830002, quorum=127.0.0.1:51231, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-14T14:52:12,005 INFO [RS:1;da1c6afcd1f9:34155 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-14T14:52:12,005 INFO [RS:1;da1c6afcd1f9:34155 {}] regionserver.HRegionServer(1031): Exiting; stopping=da1c6afcd1f9,34155,1731595913533; zookeeper connection closed. 2024-11-14T14:52:12,005 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@21beee31 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@21beee31 2024-11-14T14:52:12,008 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1731595929525.cf7ad4aad0d0e3c5a9f31c16500a8e87., hostname=da1c6afcd1f9,34155,1731595913533, seqNum=18] 2024-11-14T14:52:12,010 WARN [RPCClient-NioEventLoopGroup-6-5 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server da1c6afcd1f9:34155 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: da1c6afcd1f9/172.17.0.2:34155 Caused by: java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hbase.thirdparty.io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:336) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:339) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:52:12,010 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1731595929525.cf7ad4aad0d0e3c5a9f31c16500a8e87., hostname=da1c6afcd1f9,34155,1731595913533, seqNum=18 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1731595929525.cf7ad4aad0d0e3c5a9f31c16500a8e87., hostname=da1c6afcd1f9,34155,1731595913533, seqNum=18, error=java.net.ConnectException: Call to address=da1c6afcd1f9:34155 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: da1c6afcd1f9/172.17.0.2:34155 2024-11-14T14:52:12,010 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1731595929525.cf7ad4aad0d0e3c5a9f31c16500a8e87., hostname=da1c6afcd1f9,34155,1731595913533, seqNum=18 is java.net.ConnectException: Connection refused 2024-11-14T14:52:12,010 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testReplayEditsAfterRegionMovedWithMultiCF,,1731595929525.cf7ad4aad0d0e3c5a9f31c16500a8e87., hostname=da1c6afcd1f9,34155,1731595913533, seqNum=18 from cache 2024-11-14T14:52:12,011 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] ipc.FailedServers(52): Added failed server with address da1c6afcd1f9:34155 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: da1c6afcd1f9/172.17.0.2:34155 2024-11-14T14:52:12,087 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46007 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SplitWALCallable, pid=15 2024-11-14T14:52:12,105 INFO [RS_LOG_REPLAY_OPS-regionserver/da1c6afcd1f9:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(299): Splitting hdfs://localhost:34445/user/jenkins/test-data/35161b07-018a-0344-8c43-1519f3812567/WALs/da1c6afcd1f9,34155,1731595913533-splitting/da1c6afcd1f9%2C34155%2C1731595913533.1731595915018, size=2.2 K (2288bytes) 2024-11-14T14:52:12,105 INFO [RS_LOG_REPLAY_OPS-regionserver/da1c6afcd1f9:0-0 {event_type=RS_LOG_REPLAY, pid=15}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:34445/user/jenkins/test-data/35161b07-018a-0344-8c43-1519f3812567/WALs/da1c6afcd1f9,34155,1731595913533-splitting/da1c6afcd1f9%2C34155%2C1731595913533.1731595915018 2024-11-14T14:52:12,106 INFO [RS_LOG_REPLAY_OPS-regionserver/da1c6afcd1f9:0-0 {event_type=RS_LOG_REPLAY, pid=15}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:34445/user/jenkins/test-data/35161b07-018a-0344-8c43-1519f3812567/WALs/da1c6afcd1f9,34155,1731595913533-splitting/da1c6afcd1f9%2C34155%2C1731595913533.1731595915018 after 0ms 2024-11-14T14:52:12,133 DEBUG [RS_LOG_REPLAY_OPS-regionserver/da1c6afcd1f9:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-14T14:52:12,133 INFO [RS_LOG_REPLAY_OPS-regionserver/da1c6afcd1f9:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(310): Open hdfs://localhost:34445/user/jenkins/test-data/35161b07-018a-0344-8c43-1519f3812567/WALs/da1c6afcd1f9,34155,1731595913533-splitting/da1c6afcd1f9%2C34155%2C1731595913533.1731595915018 took 28ms 2024-11-14T14:52:12,141 DEBUG [RS_LOG_REPLAY_OPS-regionserver/da1c6afcd1f9:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(352): Last flushed sequenceid for cf7ad4aad0d0e3c5a9f31c16500a8e87: last_flushed_sequence_id: 12 store_sequence_id { family_name: "cf1" sequence_id: 12 } store_sequence_id { family_name: "cf2" sequence_id: 12 } 2024-11-14T14:52:12,141 DEBUG [RS_LOG_REPLAY_OPS-regionserver/da1c6afcd1f9:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:34445/user/jenkins/test-data/35161b07-018a-0344-8c43-1519f3812567/WALs/da1c6afcd1f9,34155,1731595913533-splitting/da1c6afcd1f9%2C34155%2C1731595913533.1731595915018 so closing down 2024-11-14T14:52:12,141 DEBUG [RS_LOG_REPLAY_OPS-regionserver/da1c6afcd1f9:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-14T14:52:12,141 INFO [RS_LOG_REPLAY_OPS-regionserver/da1c6afcd1f9:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.OutputSink(145): 3 split writer threads finished 2024-11-14T14:52:12,141 INFO [RS_LOG_REPLAY_OPS-regionserver/da1c6afcd1f9:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(425): Processed 6 edits across 0 Regions in 7 ms; skipped=6; WAL=hdfs://localhost:34445/user/jenkins/test-data/35161b07-018a-0344-8c43-1519f3812567/WALs/da1c6afcd1f9,34155,1731595913533-splitting/da1c6afcd1f9%2C34155%2C1731595913533.1731595915018, size=2.2 K, length=2288, corrupted=false, cancelled=false 2024-11-14T14:52:12,141 DEBUG [RS_LOG_REPLAY_OPS-regionserver/da1c6afcd1f9:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(428): Completed split of hdfs://localhost:34445/user/jenkins/test-data/35161b07-018a-0344-8c43-1519f3812567/WALs/da1c6afcd1f9,34155,1731595913533-splitting/da1c6afcd1f9%2C34155%2C1731595913533.1731595915018, journal: Splitting hdfs://localhost:34445/user/jenkins/test-data/35161b07-018a-0344-8c43-1519f3812567/WALs/da1c6afcd1f9,34155,1731595913533-splitting/da1c6afcd1f9%2C34155%2C1731595913533.1731595915018, size=2.2 K (2288bytes) at 1731595932105Finishing writing output for hdfs://localhost:34445/user/jenkins/test-data/35161b07-018a-0344-8c43-1519f3812567/WALs/da1c6afcd1f9,34155,1731595913533-splitting/da1c6afcd1f9%2C34155%2C1731595913533.1731595915018 so closing down at 1731595932141 (+36 ms)3 split writer threads finished at 1731595932141Processed 6 edits across 0 Regions in 7 ms; skipped=6; WAL=hdfs://localhost:34445/user/jenkins/test-data/35161b07-018a-0344-8c43-1519f3812567/WALs/da1c6afcd1f9,34155,1731595913533-splitting/da1c6afcd1f9%2C34155%2C1731595913533.1731595915018, size=2.2 K, length=2288, corrupted=false, cancelled=false at 1731595932141 2024-11-14T14:52:12,141 DEBUG [RS_LOG_REPLAY_OPS-regionserver/da1c6afcd1f9:0-0 {event_type=RS_LOG_REPLAY, pid=15}] regionserver.SplitLogWorker(218): Done splitting WAL hdfs://localhost:34445/user/jenkins/test-data/35161b07-018a-0344-8c43-1519f3812567/WALs/da1c6afcd1f9,34155,1731595913533-splitting/da1c6afcd1f9%2C34155%2C1731595913533.1731595915018 2024-11-14T14:52:12,143 DEBUG [RS_LOG_REPLAY_OPS-regionserver/da1c6afcd1f9:0-0 {event_type=RS_LOG_REPLAY, pid=15}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=15 2024-11-14T14:52:12,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42151 {}] master.HMaster(4169): Remote procedure done, pid=15 2024-11-14T14:52:12,150 INFO [PEWorker-4 {}] wal.WALSplitUtil(143): Moved hdfs://localhost:34445/user/jenkins/test-data/35161b07-018a-0344-8c43-1519f3812567/WALs/da1c6afcd1f9,34155,1731595913533-splitting/da1c6afcd1f9%2C34155%2C1731595913533.1731595915018 to hdfs://localhost:34445/user/jenkins/test-data/35161b07-018a-0344-8c43-1519f3812567/oldWALs 2024-11-14T14:52:12,152 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=15, resume processing ppid=14 2024-11-14T14:52:12,152 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, ppid=14, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure da1c6afcd1f9%2C34155%2C1731595913533.1731595915018, worker=da1c6afcd1f9,46007,1731595913586 in 224 msec 2024-11-14T14:52:12,154 DEBUG [PEWorker-5 {}] master.SplitWALManager(172): Release split WAL worker=da1c6afcd1f9,46007,1731595913586 2024-11-14T14:52:12,156 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-11-14T14:52:12,157 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; SplitWALProcedure da1c6afcd1f9%2C34155%2C1731595913533.1731595915018, worker=da1c6afcd1f9,46007,1731595913586 in 235 msec 2024-11-14T14:52:12,158 INFO [PEWorker-1 {}] master.SplitLogManager(171): hdfs://localhost:34445/user/jenkins/test-data/35161b07-018a-0344-8c43-1519f3812567/WALs/da1c6afcd1f9,34155,1731595913533-splitting dir is empty, no logs to split. 2024-11-14T14:52:12,158 INFO [PEWorker-1 {}] master.SplitWALManager(105): da1c6afcd1f9,34155,1731595913533 WAL count=0, meta=false 2024-11-14T14:52:12,158 DEBUG [PEWorker-1 {}] procedure.ServerCrashProcedure(329): Check if da1c6afcd1f9,34155,1731595913533 WAL splitting is done? wals=0, meta=false 2024-11-14T14:52:12,160 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(321): Remove WAL directory for da1c6afcd1f9,34155,1731595913533 failed, ignore...File hdfs://localhost:34445/user/jenkins/test-data/35161b07-018a-0344-8c43-1519f3812567/WALs/da1c6afcd1f9,34155,1731595913533-splitting does not exist. 2024-11-14T14:52:12,162 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=cf7ad4aad0d0e3c5a9f31c16500a8e87, ASSIGN}] 2024-11-14T14:52:12,163 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=cf7ad4aad0d0e3c5a9f31c16500a8e87, ASSIGN 2024-11-14T14:52:12,165 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=cf7ad4aad0d0e3c5a9f31c16500a8e87, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-14T14:52:12,315 DEBUG [da1c6afcd1f9:42151 {}] balancer.BalancerClusterState(204): Hosts are {da1c6afcd1f9=0} racks are {/default-rack=0} 2024-11-14T14:52:12,316 DEBUG [da1c6afcd1f9:42151 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-14T14:52:12,316 DEBUG [da1c6afcd1f9:42151 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-14T14:52:12,316 DEBUG [da1c6afcd1f9:42151 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-14T14:52:12,316 DEBUG [da1c6afcd1f9:42151 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-14T14:52:12,316 INFO [da1c6afcd1f9:42151 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-14T14:52:12,316 INFO [da1c6afcd1f9:42151 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-14T14:52:12,316 DEBUG [da1c6afcd1f9:42151 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-14T14:52:12,316 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=cf7ad4aad0d0e3c5a9f31c16500a8e87, regionState=OPENING, regionLocation=da1c6afcd1f9,37695,1731595913427 2024-11-14T14:52:12,319 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1731595929525.cf7ad4aad0d0e3c5a9f31c16500a8e87., hostname=da1c6afcd1f9,34155,1731595913533, seqNum=18] 2024-11-14T14:52:12,319 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=cf7ad4aad0d0e3c5a9f31c16500a8e87, ASSIGN because future has completed 2024-11-14T14:52:12,320 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=17, ppid=16, state=RUNNABLE, hasLock=false; OpenRegionProcedure cf7ad4aad0d0e3c5a9f31c16500a8e87, server=da1c6afcd1f9,37695,1731595913427}] 2024-11-14T14:52:12,320 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] ipc.AbstractRpcClient(357): Not trying to connect to da1c6afcd1f9:34155 this server is in the failed servers list 2024-11-14T14:52:12,321 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1731595929525.cf7ad4aad0d0e3c5a9f31c16500a8e87., hostname=da1c6afcd1f9,34155,1731595913533, seqNum=18 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1731595929525.cf7ad4aad0d0e3c5a9f31c16500a8e87., hostname=da1c6afcd1f9,34155,1731595913533, seqNum=18, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=da1c6afcd1f9:34155 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: da1c6afcd1f9:34155 2024-11-14T14:52:12,321 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1731595929525.cf7ad4aad0d0e3c5a9f31c16500a8e87., hostname=da1c6afcd1f9,34155,1731595913533, seqNum=18 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: da1c6afcd1f9:34155 2024-11-14T14:52:12,321 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testReplayEditsAfterRegionMovedWithMultiCF,,1731595929525.cf7ad4aad0d0e3c5a9f31c16500a8e87., hostname=da1c6afcd1f9,34155,1731595913533, seqNum=18 from cache 2024-11-14T14:52:12,477 INFO [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1731595929525.cf7ad4aad0d0e3c5a9f31c16500a8e87. 2024-11-14T14:52:12,477 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(7752): Opening region: {ENCODED => cf7ad4aad0d0e3c5a9f31c16500a8e87, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1731595929525.cf7ad4aad0d0e3c5a9f31c16500a8e87.', STARTKEY => '', ENDKEY => ''} 2024-11-14T14:52:12,477 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF cf7ad4aad0d0e3c5a9f31c16500a8e87 2024-11-14T14:52:12,477 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1731595929525.cf7ad4aad0d0e3c5a9f31c16500a8e87.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T14:52:12,478 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(7794): checking encryption for cf7ad4aad0d0e3c5a9f31c16500a8e87 2024-11-14T14:52:12,478 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(7797): checking classloading for cf7ad4aad0d0e3c5a9f31c16500a8e87 2024-11-14T14:52:12,479 INFO [StoreOpener-cf7ad4aad0d0e3c5a9f31c16500a8e87-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region cf7ad4aad0d0e3c5a9f31c16500a8e87 2024-11-14T14:52:12,480 INFO [StoreOpener-cf7ad4aad0d0e3c5a9f31c16500a8e87-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cf7ad4aad0d0e3c5a9f31c16500a8e87 columnFamilyName cf1 2024-11-14T14:52:12,480 DEBUG [StoreOpener-cf7ad4aad0d0e3c5a9f31c16500a8e87-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T14:52:12,486 DEBUG [StoreOpener-cf7ad4aad0d0e3c5a9f31c16500a8e87-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:34445/user/jenkins/test-data/35161b07-018a-0344-8c43-1519f3812567/data/default/testReplayEditsAfterRegionMovedWithMultiCF/cf7ad4aad0d0e3c5a9f31c16500a8e87/cf1/efdb7fed42594e78b882b0ad2e13a8c1 2024-11-14T14:52:12,487 INFO [StoreOpener-cf7ad4aad0d0e3c5a9f31c16500a8e87-1 {}] regionserver.HStore(327): Store=cf7ad4aad0d0e3c5a9f31c16500a8e87/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T14:52:12,487 INFO [StoreOpener-cf7ad4aad0d0e3c5a9f31c16500a8e87-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region cf7ad4aad0d0e3c5a9f31c16500a8e87 2024-11-14T14:52:12,487 INFO [StoreOpener-cf7ad4aad0d0e3c5a9f31c16500a8e87-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cf7ad4aad0d0e3c5a9f31c16500a8e87 columnFamilyName cf2 2024-11-14T14:52:12,488 DEBUG [StoreOpener-cf7ad4aad0d0e3c5a9f31c16500a8e87-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T14:52:12,494 DEBUG [StoreOpener-cf7ad4aad0d0e3c5a9f31c16500a8e87-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:34445/user/jenkins/test-data/35161b07-018a-0344-8c43-1519f3812567/data/default/testReplayEditsAfterRegionMovedWithMultiCF/cf7ad4aad0d0e3c5a9f31c16500a8e87/cf2/2b8c1aa97dce4f2aa88a5be7c980475c 2024-11-14T14:52:12,494 INFO [StoreOpener-cf7ad4aad0d0e3c5a9f31c16500a8e87-1 {}] regionserver.HStore(327): Store=cf7ad4aad0d0e3c5a9f31c16500a8e87/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T14:52:12,494 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1038): replaying wal for cf7ad4aad0d0e3c5a9f31c16500a8e87 2024-11-14T14:52:12,495 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34445/user/jenkins/test-data/35161b07-018a-0344-8c43-1519f3812567/data/default/testReplayEditsAfterRegionMovedWithMultiCF/cf7ad4aad0d0e3c5a9f31c16500a8e87 2024-11-14T14:52:12,496 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34445/user/jenkins/test-data/35161b07-018a-0344-8c43-1519f3812567/data/default/testReplayEditsAfterRegionMovedWithMultiCF/cf7ad4aad0d0e3c5a9f31c16500a8e87 2024-11-14T14:52:12,496 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1048): stopping wal replay for cf7ad4aad0d0e3c5a9f31c16500a8e87 2024-11-14T14:52:12,496 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1060): Cleaning up temporary data for cf7ad4aad0d0e3c5a9f31c16500a8e87 2024-11-14T14:52:12,497 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-11-14T14:52:12,498 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1093): writing seq id for cf7ad4aad0d0e3c5a9f31c16500a8e87 2024-11-14T14:52:12,499 INFO [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1114): Opened cf7ad4aad0d0e3c5a9f31c16500a8e87; next sequenceid=18; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72322564, jitterRate=0.07769018411636353}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-11-14T14:52:12,499 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1122): Running coprocessor post-open hooks for cf7ad4aad0d0e3c5a9f31c16500a8e87 2024-11-14T14:52:12,500 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1006): Region open journal for cf7ad4aad0d0e3c5a9f31c16500a8e87: Running coprocessor pre-open hook at 1731595932478Writing region info on filesystem at 1731595932478Initializing all the Stores at 1731595932479 (+1 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731595932479Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731595932479Cleaning up temporary data from old regions at 1731595932496 (+17 ms)Running coprocessor post-open hooks at 1731595932499 (+3 ms)Region opened successfully at 1731595932500 (+1 ms) 2024-11-14T14:52:12,501 INFO [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1731595929525.cf7ad4aad0d0e3c5a9f31c16500a8e87., pid=17, masterSystemTime=1731595932472 2024-11-14T14:52:12,503 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1731595929525.cf7ad4aad0d0e3c5a9f31c16500a8e87. 2024-11-14T14:52:12,503 INFO [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1731595929525.cf7ad4aad0d0e3c5a9f31c16500a8e87. 2024-11-14T14:52:12,504 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=cf7ad4aad0d0e3c5a9f31c16500a8e87, regionState=OPEN, openSeqNum=18, regionLocation=da1c6afcd1f9,37695,1731595913427 2024-11-14T14:52:12,506 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=17, ppid=16, state=RUNNABLE, hasLock=false; OpenRegionProcedure cf7ad4aad0d0e3c5a9f31c16500a8e87, server=da1c6afcd1f9,37695,1731595913427 because future has completed 2024-11-14T14:52:12,510 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=17, resume processing ppid=16 2024-11-14T14:52:12,510 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, ppid=16, state=SUCCESS, hasLock=false; OpenRegionProcedure cf7ad4aad0d0e3c5a9f31c16500a8e87, server=da1c6afcd1f9,37695,1731595913427 in 187 msec 2024-11-14T14:52:12,512 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=13 2024-11-14T14:52:12,512 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=13, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=cf7ad4aad0d0e3c5a9f31c16500a8e87, ASSIGN in 348 msec 2024-11-14T14:52:12,513 INFO [PEWorker-4 {}] procedure.ServerCrashProcedure(291): removed crashed server da1c6afcd1f9,34155,1731595913533 after splitting done 2024-11-14T14:52:12,515 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; ServerCrashProcedure da1c6afcd1f9,34155,1731595913533, splitWal=true, meta=false in 608 msec 2024-11-14T14:52:12,838 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1731595929525.cf7ad4aad0d0e3c5a9f31c16500a8e87., hostname=da1c6afcd1f9,37695,1731595913427, seqNum=18] 2024-11-14T14:52:12,853 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsAfterRegionMovedWithMultiCF Thread=410 (was 414), OpenFileDescriptor=1048 (was 1023) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=186 (was 176) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=7582 (was 7613) 2024-11-14T14:52:12,855 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1048 is superior to 1024 2024-11-14T14:52:12,867 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsAfterPartialFlush Thread=410, OpenFileDescriptor=1048, MaxFileDescriptor=1048576, SystemLoadAverage=186, ProcessCount=11, AvailableMemoryMB=7581 2024-11-14T14:52:12,867 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1048 is superior to 1024 2024-11-14T14:52:12,882 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-14T14:52:12,884 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-14T14:52:12,885 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-14T14:52:12,888 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-51656481, suffix=, logDir=hdfs://localhost:34445/hbase/WALs/hregion-51656481, archiveDir=hdfs://localhost:34445/hbase/oldWALs, maxLogs=32 2024-11-14T14:52:12,901 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-51656481/hregion-51656481.1731595932888, exclude list is [], retry=0 2024-11-14T14:52:12,904 DEBUG [AsyncFSWAL-20-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42819,DS-c1b8ccdc-5e2c-4b3d-bb1e-6941b40dfb7e,DISK] 2024-11-14T14:52:12,904 DEBUG [AsyncFSWAL-20-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44549,DS-913adb4a-7946-4f46-896c-56bcdc4fcfa8,DISK] 2024-11-14T14:52:12,904 DEBUG [AsyncFSWAL-20-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36189,DS-5c245456-9209-4937-bfad-43a9a1b09820,DISK] 2024-11-14T14:52:12,932 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-51656481/hregion-51656481.1731595932888 2024-11-14T14:52:12,933 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34777:34777),(127.0.0.1/127.0.0.1:33515:33515),(127.0.0.1/127.0.0.1:33623:33623)] 2024-11-14T14:52:12,933 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => b525c5f69b8d59a438788e12849dfa00, NAME => 'testReplayEditsWrittenViaHRegion,,1731595932883.b525c5f69b8d59a438788e12849dfa00.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenViaHRegion', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34445/hbase 2024-11-14T14:52:12,946 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42819 is added to blk_1073741893_1071 (size=67) 2024-11-14T14:52:12,946 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36189 is added to blk_1073741893_1071 (size=67) 2024-11-14T14:52:12,947 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44549 is added to blk_1073741893_1071 (size=67) 2024-11-14T14:52:12,948 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1731595932883.b525c5f69b8d59a438788e12849dfa00.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T14:52:12,949 INFO [StoreOpener-b525c5f69b8d59a438788e12849dfa00-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region b525c5f69b8d59a438788e12849dfa00 2024-11-14T14:52:12,951 INFO [StoreOpener-b525c5f69b8d59a438788e12849dfa00-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b525c5f69b8d59a438788e12849dfa00 columnFamilyName a 2024-11-14T14:52:12,951 DEBUG [StoreOpener-b525c5f69b8d59a438788e12849dfa00-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T14:52:12,951 INFO [StoreOpener-b525c5f69b8d59a438788e12849dfa00-1 {}] regionserver.HStore(327): Store=b525c5f69b8d59a438788e12849dfa00/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T14:52:12,951 INFO [StoreOpener-b525c5f69b8d59a438788e12849dfa00-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region b525c5f69b8d59a438788e12849dfa00 2024-11-14T14:52:12,953 INFO [StoreOpener-b525c5f69b8d59a438788e12849dfa00-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b525c5f69b8d59a438788e12849dfa00 columnFamilyName b 2024-11-14T14:52:12,953 DEBUG [StoreOpener-b525c5f69b8d59a438788e12849dfa00-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T14:52:12,954 INFO [StoreOpener-b525c5f69b8d59a438788e12849dfa00-1 {}] regionserver.HStore(327): Store=b525c5f69b8d59a438788e12849dfa00/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T14:52:12,954 INFO [StoreOpener-b525c5f69b8d59a438788e12849dfa00-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region b525c5f69b8d59a438788e12849dfa00 2024-11-14T14:52:12,955 INFO [StoreOpener-b525c5f69b8d59a438788e12849dfa00-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b525c5f69b8d59a438788e12849dfa00 columnFamilyName c 2024-11-14T14:52:12,955 DEBUG [StoreOpener-b525c5f69b8d59a438788e12849dfa00-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T14:52:12,955 INFO [StoreOpener-b525c5f69b8d59a438788e12849dfa00-1 {}] regionserver.HStore(327): Store=b525c5f69b8d59a438788e12849dfa00/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T14:52:12,956 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for b525c5f69b8d59a438788e12849dfa00 2024-11-14T14:52:12,956 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34445/hbase/data/default/testReplayEditsWrittenViaHRegion/b525c5f69b8d59a438788e12849dfa00 2024-11-14T14:52:12,956 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34445/hbase/data/default/testReplayEditsWrittenViaHRegion/b525c5f69b8d59a438788e12849dfa00 2024-11-14T14:52:12,957 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for b525c5f69b8d59a438788e12849dfa00 2024-11-14T14:52:12,958 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for b525c5f69b8d59a438788e12849dfa00 2024-11-14T14:52:12,958 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-14T14:52:12,959 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for b525c5f69b8d59a438788e12849dfa00 2024-11-14T14:52:12,962 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34445/hbase/data/default/testReplayEditsWrittenViaHRegion/b525c5f69b8d59a438788e12849dfa00/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-14T14:52:12,962 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened b525c5f69b8d59a438788e12849dfa00; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65637902, jitterRate=-0.021919041872024536}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-14T14:52:12,963 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for b525c5f69b8d59a438788e12849dfa00: Writing region info on filesystem at 1731595932948Initializing all the Stores at 1731595932949 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731595932949Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731595932949Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731595932949Cleaning up temporary data from old regions at 1731595932958 (+9 ms)Region opened successfully at 1731595932963 (+5 ms) 2024-11-14T14:52:12,963 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing b525c5f69b8d59a438788e12849dfa00, disabling compactions & flushes 2024-11-14T14:52:12,963 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1731595932883.b525c5f69b8d59a438788e12849dfa00. 2024-11-14T14:52:12,963 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1731595932883.b525c5f69b8d59a438788e12849dfa00. 2024-11-14T14:52:12,963 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1731595932883.b525c5f69b8d59a438788e12849dfa00. after waiting 0 ms 2024-11-14T14:52:12,963 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1731595932883.b525c5f69b8d59a438788e12849dfa00. 2024-11-14T14:52:12,963 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1731595932883.b525c5f69b8d59a438788e12849dfa00. 2024-11-14T14:52:12,964 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for b525c5f69b8d59a438788e12849dfa00: Waiting for close lock at 1731595932963Disabling compacts and flushes for region at 1731595932963Disabling writes for close at 1731595932963Writing region close event to WAL at 1731595932963Closed at 1731595932963 2024-11-14T14:52:12,966 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36189 is added to blk_1073741892_1070 (size=175) 2024-11-14T14:52:12,966 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42819 is added to blk_1073741892_1070 (size=175) 2024-11-14T14:52:12,966 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44549 is added to blk_1073741892_1070 (size=175) 2024-11-14T14:52:12,968 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-14T14:52:12,968 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-51656481:(num 1731595932888) 2024-11-14T14:52:12,968 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-14T14:52:12,970 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:34445/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731595932881, archiveDir=hdfs://localhost:34445/hbase/oldWALs, maxLogs=32 2024-11-14T14:52:12,983 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731595932881/wal.1731595932970, exclude list is [], retry=0 2024-11-14T14:52:12,986 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42819,DS-c1b8ccdc-5e2c-4b3d-bb1e-6941b40dfb7e,DISK] 2024-11-14T14:52:12,986 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36189,DS-5c245456-9209-4937-bfad-43a9a1b09820,DISK] 2024-11-14T14:52:12,987 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44549,DS-913adb4a-7946-4f46-896c-56bcdc4fcfa8,DISK] 2024-11-14T14:52:13,013 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731595932881/wal.1731595932970 2024-11-14T14:52:13,014 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34777:34777),(127.0.0.1/127.0.0.1:33623:33623),(127.0.0.1/127.0.0.1:33515:33515)] 2024-11-14T14:52:13,014 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => b525c5f69b8d59a438788e12849dfa00, NAME => 'testReplayEditsWrittenViaHRegion,,1731595932883.b525c5f69b8d59a438788e12849dfa00.', STARTKEY => '', ENDKEY => ''} 2024-11-14T14:52:13,014 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1731595932883.b525c5f69b8d59a438788e12849dfa00.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T14:52:13,014 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for b525c5f69b8d59a438788e12849dfa00 2024-11-14T14:52:13,014 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for b525c5f69b8d59a438788e12849dfa00 2024-11-14T14:52:13,016 INFO [StoreOpener-b525c5f69b8d59a438788e12849dfa00-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region b525c5f69b8d59a438788e12849dfa00 2024-11-14T14:52:13,017 INFO [StoreOpener-b525c5f69b8d59a438788e12849dfa00-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b525c5f69b8d59a438788e12849dfa00 columnFamilyName a 2024-11-14T14:52:13,017 DEBUG [StoreOpener-b525c5f69b8d59a438788e12849dfa00-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T14:52:13,017 INFO [StoreOpener-b525c5f69b8d59a438788e12849dfa00-1 {}] regionserver.HStore(327): Store=b525c5f69b8d59a438788e12849dfa00/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T14:52:13,017 INFO [StoreOpener-b525c5f69b8d59a438788e12849dfa00-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region b525c5f69b8d59a438788e12849dfa00 2024-11-14T14:52:13,018 INFO [StoreOpener-b525c5f69b8d59a438788e12849dfa00-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b525c5f69b8d59a438788e12849dfa00 columnFamilyName b 2024-11-14T14:52:13,018 DEBUG [StoreOpener-b525c5f69b8d59a438788e12849dfa00-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T14:52:13,019 INFO [StoreOpener-b525c5f69b8d59a438788e12849dfa00-1 {}] regionserver.HStore(327): Store=b525c5f69b8d59a438788e12849dfa00/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T14:52:13,019 INFO [StoreOpener-b525c5f69b8d59a438788e12849dfa00-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region b525c5f69b8d59a438788e12849dfa00 2024-11-14T14:52:13,019 INFO [StoreOpener-b525c5f69b8d59a438788e12849dfa00-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b525c5f69b8d59a438788e12849dfa00 columnFamilyName c 2024-11-14T14:52:13,019 DEBUG [StoreOpener-b525c5f69b8d59a438788e12849dfa00-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T14:52:13,020 INFO [StoreOpener-b525c5f69b8d59a438788e12849dfa00-1 {}] regionserver.HStore(327): Store=b525c5f69b8d59a438788e12849dfa00/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T14:52:13,020 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for b525c5f69b8d59a438788e12849dfa00 2024-11-14T14:52:13,021 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34445/hbase/data/default/testReplayEditsWrittenViaHRegion/b525c5f69b8d59a438788e12849dfa00 2024-11-14T14:52:13,022 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34445/hbase/data/default/testReplayEditsWrittenViaHRegion/b525c5f69b8d59a438788e12849dfa00 2024-11-14T14:52:13,023 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for b525c5f69b8d59a438788e12849dfa00 2024-11-14T14:52:13,023 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for b525c5f69b8d59a438788e12849dfa00 2024-11-14T14:52:13,023 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-14T14:52:13,024 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for b525c5f69b8d59a438788e12849dfa00 2024-11-14T14:52:13,025 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened b525c5f69b8d59a438788e12849dfa00; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64251644, jitterRate=-0.0425758957862854}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-14T14:52:13,026 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for b525c5f69b8d59a438788e12849dfa00: Writing region info on filesystem at 1731595933014Initializing all the Stores at 1731595933015 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731595933015Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731595933016 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731595933016Cleaning up temporary data from old regions at 1731595933023 (+7 ms)Region opened successfully at 1731595933026 (+3 ms) 2024-11-14T14:52:13,056 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing b525c5f69b8d59a438788e12849dfa00 3/3 column families, dataSize=2.55 KB heapSize=5.44 KB 2024-11-14T14:52:13,074 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/hbase/data/default/testReplayEditsWrittenViaHRegion/b525c5f69b8d59a438788e12849dfa00/.tmp/a/a9cd9c6c49b24bf7955c0eba1d24e061 is 91, key is testReplayEditsWrittenViaHRegion/a:x0/1731595933026/Put/seqid=0 2024-11-14T14:52:13,081 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36189 is added to blk_1073741895_1073 (size=5958) 2024-11-14T14:52:13,081 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42819 is added to blk_1073741895_1073 (size=5958) 2024-11-14T14:52:13,082 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44549 is added to blk_1073741895_1073 (size=5958) 2024-11-14T14:52:13,083 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:34445/hbase/data/default/testReplayEditsWrittenViaHRegion/b525c5f69b8d59a438788e12849dfa00/.tmp/a/a9cd9c6c49b24bf7955c0eba1d24e061 2024-11-14T14:52:13,107 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/hbase/data/default/testReplayEditsWrittenViaHRegion/b525c5f69b8d59a438788e12849dfa00/.tmp/b/d93a8d257d284dc190bcf1c92839bbc3 is 91, key is testReplayEditsWrittenViaHRegion/b:x0/1731595933034/Put/seqid=0 2024-11-14T14:52:13,113 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44549 is added to blk_1073741896_1074 (size=5958) 2024-11-14T14:52:13,114 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42819 is added to blk_1073741896_1074 (size=5958) 2024-11-14T14:52:13,114 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36189 is added to blk_1073741896_1074 (size=5958) 2024-11-14T14:52:13,115 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:34445/hbase/data/default/testReplayEditsWrittenViaHRegion/b525c5f69b8d59a438788e12849dfa00/.tmp/b/d93a8d257d284dc190bcf1c92839bbc3 2024-11-14T14:52:13,134 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/hbase/data/default/testReplayEditsWrittenViaHRegion/b525c5f69b8d59a438788e12849dfa00/.tmp/c/27bc2a3e01aa43f89a84e1edfc466c7e is 91, key is testReplayEditsWrittenViaHRegion/c:x0/1731595933043/Put/seqid=0 2024-11-14T14:52:13,140 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42819 is added to blk_1073741897_1075 (size=5958) 2024-11-14T14:52:13,141 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36189 is added to blk_1073741897_1075 (size=5958) 2024-11-14T14:52:13,141 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44549 is added to blk_1073741897_1075 (size=5958) 2024-11-14T14:52:13,142 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:34445/hbase/data/default/testReplayEditsWrittenViaHRegion/b525c5f69b8d59a438788e12849dfa00/.tmp/c/27bc2a3e01aa43f89a84e1edfc466c7e 2024-11-14T14:52:13,148 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/hbase/data/default/testReplayEditsWrittenViaHRegion/b525c5f69b8d59a438788e12849dfa00/.tmp/a/a9cd9c6c49b24bf7955c0eba1d24e061 as hdfs://localhost:34445/hbase/data/default/testReplayEditsWrittenViaHRegion/b525c5f69b8d59a438788e12849dfa00/a/a9cd9c6c49b24bf7955c0eba1d24e061 2024-11-14T14:52:13,152 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenViaHRegion 2024-11-14T14:52:13,152 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenViaHRegion Metrics about Tables on a single HBase RegionServer 2024-11-14T14:52:13,154 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/hbase/data/default/testReplayEditsWrittenViaHRegion/b525c5f69b8d59a438788e12849dfa00/a/a9cd9c6c49b24bf7955c0eba1d24e061, entries=10, sequenceid=33, filesize=5.8 K 2024-11-14T14:52:13,154 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testSequentialEditLogSeqNum 2024-11-14T14:52:13,154 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testSequentialEditLogSeqNum Metrics about Tables on a single HBase RegionServer 2024-11-14T14:52:13,155 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/hbase/data/default/testReplayEditsWrittenViaHRegion/b525c5f69b8d59a438788e12849dfa00/.tmp/b/d93a8d257d284dc190bcf1c92839bbc3 as hdfs://localhost:34445/hbase/data/default/testReplayEditsWrittenViaHRegion/b525c5f69b8d59a438788e12849dfa00/b/d93a8d257d284dc190bcf1c92839bbc3 2024-11-14T14:52:13,155 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterRegionMovedWithMultiCF 2024-11-14T14:52:13,155 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterRegionMovedWithMultiCF Metrics about Tables on a single HBase RegionServer 2024-11-14T14:52:13,156 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRegionMadeOfBulkLoadedFilesOnly 2024-11-14T14:52:13,156 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testRegionMadeOfBulkLoadedFilesOnly Metrics about Tables on a single HBase RegionServer 2024-11-14T14:52:13,160 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/hbase/data/default/testReplayEditsWrittenViaHRegion/b525c5f69b8d59a438788e12849dfa00/b/d93a8d257d284dc190bcf1c92839bbc3, entries=10, sequenceid=33, filesize=5.8 K 2024-11-14T14:52:13,161 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/hbase/data/default/testReplayEditsWrittenViaHRegion/b525c5f69b8d59a438788e12849dfa00/.tmp/c/27bc2a3e01aa43f89a84e1edfc466c7e as hdfs://localhost:34445/hbase/data/default/testReplayEditsWrittenViaHRegion/b525c5f69b8d59a438788e12849dfa00/c/27bc2a3e01aa43f89a84e1edfc466c7e 2024-11-14T14:52:13,167 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/hbase/data/default/testReplayEditsWrittenViaHRegion/b525c5f69b8d59a438788e12849dfa00/c/27bc2a3e01aa43f89a84e1edfc466c7e, entries=10, sequenceid=33, filesize=5.8 K 2024-11-14T14:52:13,168 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~2.55 KB/2610, heapSize ~5.39 KB/5520, currentSize=0 B/0 for b525c5f69b8d59a438788e12849dfa00 in 112ms, sequenceid=33, compaction requested=false 2024-11-14T14:52:13,168 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for b525c5f69b8d59a438788e12849dfa00: 2024-11-14T14:52:13,168 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing b525c5f69b8d59a438788e12849dfa00, disabling compactions & flushes 2024-11-14T14:52:13,168 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1731595932883.b525c5f69b8d59a438788e12849dfa00. 2024-11-14T14:52:13,168 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1731595932883.b525c5f69b8d59a438788e12849dfa00. 2024-11-14T14:52:13,168 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1731595932883.b525c5f69b8d59a438788e12849dfa00. after waiting 0 ms 2024-11-14T14:52:13,168 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1731595932883.b525c5f69b8d59a438788e12849dfa00. 2024-11-14T14:52:13,170 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1731595932883.b525c5f69b8d59a438788e12849dfa00. 2024-11-14T14:52:13,170 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for b525c5f69b8d59a438788e12849dfa00: Waiting for close lock at 1731595933168Disabling compacts and flushes for region at 1731595933168Disabling writes for close at 1731595933168Writing region close event to WAL at 1731595933170 (+2 ms)Closed at 1731595933170 2024-11-14T14:52:13,173 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42819 is added to blk_1073741894_1072 (size=6311) 2024-11-14T14:52:13,173 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44549 is added to blk_1073741894_1072 (size=6311) 2024-11-14T14:52:13,174 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36189 is added to blk_1073741894_1072 (size=6311) 2024-11-14T14:52:13,178 DEBUG [Time-limited test {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34445/hbase/data/default/testReplayEditsWrittenViaHRegion/b525c5f69b8d59a438788e12849dfa00/b/d93a8d257d284dc190bcf1c92839bbc3 to hdfs://localhost:34445/hbase/archive/data/default/testReplayEditsWrittenViaHRegion/b525c5f69b8d59a438788e12849dfa00/b/d93a8d257d284dc190bcf1c92839bbc3 2024-11-14T14:52:13,195 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:34445/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731595932881/wal.1731595932970, size=6.2 K (6311bytes) 2024-11-14T14:52:13,195 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:34445/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731595932881/wal.1731595932970 2024-11-14T14:52:13,195 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:34445/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731595932881/wal.1731595932970 after 0ms 2024-11-14T14:52:13,225 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-14T14:52:13,225 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:34445/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731595932881/wal.1731595932970 took 31ms 2024-11-14T14:52:13,228 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:34445/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731595932881/wal.1731595932970 so closing down 2024-11-14T14:52:13,229 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-14T14:52:13,230 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1731595932970.temp 2024-11-14T14:52:13,257 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:34445/hbase/data/default/testReplayEditsWrittenViaHRegion/b525c5f69b8d59a438788e12849dfa00/recovered.edits/0000000000000000003-wal.1731595932970.temp 2024-11-14T14:52:13,258 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-14T14:52:13,264 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42819 is added to blk_1073741898_1076 (size=5575) 2024-11-14T14:52:13,265 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36189 is added to blk_1073741898_1076 (size=5575) 2024-11-14T14:52:13,265 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44549 is added to blk_1073741898_1076 (size=5575) 2024-11-14T14:52:13,266 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:34445/hbase/data/default/testReplayEditsWrittenViaHRegion/b525c5f69b8d59a438788e12849dfa00/recovered.edits/0000000000000000003-wal.1731595932970.temp (wrote 30 edits, skipped 0 edits in 0 ms) 2024-11-14T14:52:13,267 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:34445/hbase/data/default/testReplayEditsWrittenViaHRegion/b525c5f69b8d59a438788e12849dfa00/recovered.edits/0000000000000000003-wal.1731595932970.temp to hdfs://localhost:34445/hbase/data/default/testReplayEditsWrittenViaHRegion/b525c5f69b8d59a438788e12849dfa00/recovered.edits/0000000000000000032 2024-11-14T14:52:13,268 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 32 edits across 1 Regions in 42 ms; skipped=2; WAL=hdfs://localhost:34445/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731595932881/wal.1731595932970, size=6.2 K, length=6311, corrupted=false, cancelled=false 2024-11-14T14:52:13,268 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:34445/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731595932881/wal.1731595932970, journal: Splitting hdfs://localhost:34445/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731595932881/wal.1731595932970, size=6.2 K (6311bytes) at 1731595933195Finishing writing output for hdfs://localhost:34445/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731595932881/wal.1731595932970 so closing down at 1731595933229 (+34 ms)Creating recovered edits writer path=hdfs://localhost:34445/hbase/data/default/testReplayEditsWrittenViaHRegion/b525c5f69b8d59a438788e12849dfa00/recovered.edits/0000000000000000003-wal.1731595932970.temp at 1731595933257 (+28 ms)3 split writer threads finished at 1731595933258 (+1 ms)Closed recovered edits writer path=hdfs://localhost:34445/hbase/data/default/testReplayEditsWrittenViaHRegion/b525c5f69b8d59a438788e12849dfa00/recovered.edits/0000000000000000003-wal.1731595932970.temp (wrote 30 edits, skipped 0 edits in 0 ms) at 1731595933266 (+8 ms)Rename recovered edits hdfs://localhost:34445/hbase/data/default/testReplayEditsWrittenViaHRegion/b525c5f69b8d59a438788e12849dfa00/recovered.edits/0000000000000000003-wal.1731595932970.temp to hdfs://localhost:34445/hbase/data/default/testReplayEditsWrittenViaHRegion/b525c5f69b8d59a438788e12849dfa00/recovered.edits/0000000000000000032 at 1731595933267 (+1 ms)Processed 32 edits across 1 Regions in 42 ms; skipped=2; WAL=hdfs://localhost:34445/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731595932881/wal.1731595932970, size=6.2 K, length=6311, corrupted=false, cancelled=false at 1731595933268 (+1 ms) 2024-11-14T14:52:13,269 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:34445/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731595932881/wal.1731595932970 to hdfs://localhost:34445/hbase/oldWALs/wal.1731595932970 2024-11-14T14:52:13,270 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:34445/hbase/data/default/testReplayEditsWrittenViaHRegion/b525c5f69b8d59a438788e12849dfa00/recovered.edits/0000000000000000032 2024-11-14T14:52:13,270 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-14T14:52:13,272 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:34445/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731595932881, archiveDir=hdfs://localhost:34445/hbase/oldWALs, maxLogs=32 2024-11-14T14:52:13,285 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731595932881/wal.1731595933272, exclude list is [], retry=0 2024-11-14T14:52:13,288 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44549,DS-913adb4a-7946-4f46-896c-56bcdc4fcfa8,DISK] 2024-11-14T14:52:13,289 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36189,DS-5c245456-9209-4937-bfad-43a9a1b09820,DISK] 2024-11-14T14:52:13,289 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42819,DS-c1b8ccdc-5e2c-4b3d-bb1e-6941b40dfb7e,DISK] 2024-11-14T14:52:13,317 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731595932881/wal.1731595933272 2024-11-14T14:52:13,317 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33515:33515),(127.0.0.1/127.0.0.1:33623:33623),(127.0.0.1/127.0.0.1:34777:34777)] 2024-11-14T14:52:13,318 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => b525c5f69b8d59a438788e12849dfa00, NAME => 'testReplayEditsWrittenViaHRegion,,1731595932883.b525c5f69b8d59a438788e12849dfa00.', STARTKEY => '', ENDKEY => ''} 2024-11-14T14:52:13,318 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1731595932883.b525c5f69b8d59a438788e12849dfa00.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T14:52:13,318 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for b525c5f69b8d59a438788e12849dfa00 2024-11-14T14:52:13,318 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for b525c5f69b8d59a438788e12849dfa00 2024-11-14T14:52:13,319 INFO [StoreOpener-b525c5f69b8d59a438788e12849dfa00-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region b525c5f69b8d59a438788e12849dfa00 2024-11-14T14:52:13,320 INFO [StoreOpener-b525c5f69b8d59a438788e12849dfa00-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b525c5f69b8d59a438788e12849dfa00 columnFamilyName a 2024-11-14T14:52:13,320 DEBUG [StoreOpener-b525c5f69b8d59a438788e12849dfa00-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T14:52:13,327 DEBUG [StoreOpener-b525c5f69b8d59a438788e12849dfa00-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:34445/hbase/data/default/testReplayEditsWrittenViaHRegion/b525c5f69b8d59a438788e12849dfa00/a/a9cd9c6c49b24bf7955c0eba1d24e061 2024-11-14T14:52:13,327 INFO [StoreOpener-b525c5f69b8d59a438788e12849dfa00-1 {}] regionserver.HStore(327): Store=b525c5f69b8d59a438788e12849dfa00/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T14:52:13,327 INFO [StoreOpener-b525c5f69b8d59a438788e12849dfa00-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region b525c5f69b8d59a438788e12849dfa00 2024-11-14T14:52:13,328 INFO [StoreOpener-b525c5f69b8d59a438788e12849dfa00-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b525c5f69b8d59a438788e12849dfa00 columnFamilyName b 2024-11-14T14:52:13,328 DEBUG [StoreOpener-b525c5f69b8d59a438788e12849dfa00-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T14:52:13,329 INFO [StoreOpener-b525c5f69b8d59a438788e12849dfa00-1 {}] regionserver.HStore(327): Store=b525c5f69b8d59a438788e12849dfa00/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T14:52:13,329 INFO [StoreOpener-b525c5f69b8d59a438788e12849dfa00-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region b525c5f69b8d59a438788e12849dfa00 2024-11-14T14:52:13,330 INFO [StoreOpener-b525c5f69b8d59a438788e12849dfa00-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b525c5f69b8d59a438788e12849dfa00 columnFamilyName c 2024-11-14T14:52:13,330 DEBUG [StoreOpener-b525c5f69b8d59a438788e12849dfa00-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T14:52:13,335 DEBUG [StoreOpener-b525c5f69b8d59a438788e12849dfa00-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:34445/hbase/data/default/testReplayEditsWrittenViaHRegion/b525c5f69b8d59a438788e12849dfa00/c/27bc2a3e01aa43f89a84e1edfc466c7e 2024-11-14T14:52:13,336 INFO [StoreOpener-b525c5f69b8d59a438788e12849dfa00-1 {}] regionserver.HStore(327): Store=b525c5f69b8d59a438788e12849dfa00/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T14:52:13,336 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for b525c5f69b8d59a438788e12849dfa00 2024-11-14T14:52:13,337 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34445/hbase/data/default/testReplayEditsWrittenViaHRegion/b525c5f69b8d59a438788e12849dfa00 2024-11-14T14:52:13,338 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:34445/hbase/data/default/testReplayEditsWrittenViaHRegion/b525c5f69b8d59a438788e12849dfa00 2024-11-14T14:52:13,338 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:34445/hbase/data/default/testReplayEditsWrittenViaHRegion/b525c5f69b8d59a438788e12849dfa00/recovered.edits/0000000000000000032 2024-11-14T14:52:13,369 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-14T14:52:13,371 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 10, skipped 20, firstSequenceIdInLog=3, maxSequenceIdInLog=32, path=hdfs://localhost:34445/hbase/data/default/testReplayEditsWrittenViaHRegion/b525c5f69b8d59a438788e12849dfa00/recovered.edits/0000000000000000032 2024-11-14T14:52:13,371 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing b525c5f69b8d59a438788e12849dfa00 3/3 column families, dataSize=870 B heapSize=2.31 KB 2024-11-14T14:52:13,394 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/hbase/data/default/testReplayEditsWrittenViaHRegion/b525c5f69b8d59a438788e12849dfa00/.tmp/b/39659817f852480bbdeeddc3180b8669 is 91, key is testReplayEditsWrittenViaHRegion/b:x0/1731595933034/Put/seqid=0 2024-11-14T14:52:13,402 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44549 is added to blk_1073741900_1078 (size=5958) 2024-11-14T14:52:13,402 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36189 is added to blk_1073741900_1078 (size=5958) 2024-11-14T14:52:13,402 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42819 is added to blk_1073741900_1078 (size=5958) 2024-11-14T14:52:13,403 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=32 (bloomFilter=true), to=hdfs://localhost:34445/hbase/data/default/testReplayEditsWrittenViaHRegion/b525c5f69b8d59a438788e12849dfa00/.tmp/b/39659817f852480bbdeeddc3180b8669 2024-11-14T14:52:13,408 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/hbase/data/default/testReplayEditsWrittenViaHRegion/b525c5f69b8d59a438788e12849dfa00/.tmp/b/39659817f852480bbdeeddc3180b8669 as hdfs://localhost:34445/hbase/data/default/testReplayEditsWrittenViaHRegion/b525c5f69b8d59a438788e12849dfa00/b/39659817f852480bbdeeddc3180b8669 2024-11-14T14:52:13,413 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/hbase/data/default/testReplayEditsWrittenViaHRegion/b525c5f69b8d59a438788e12849dfa00/b/39659817f852480bbdeeddc3180b8669, entries=10, sequenceid=32, filesize=5.8 K 2024-11-14T14:52:13,414 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~870 B/870, heapSize ~1.80 KB/1840, currentSize=0 B/0 for b525c5f69b8d59a438788e12849dfa00 in 43ms, sequenceid=32, compaction requested=false; wal=null 2024-11-14T14:52:13,414 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:34445/hbase/data/default/testReplayEditsWrittenViaHRegion/b525c5f69b8d59a438788e12849dfa00/recovered.edits/0000000000000000032 2024-11-14T14:52:13,415 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for b525c5f69b8d59a438788e12849dfa00 2024-11-14T14:52:13,415 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for b525c5f69b8d59a438788e12849dfa00 2024-11-14T14:52:13,416 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-14T14:52:13,417 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for b525c5f69b8d59a438788e12849dfa00 2024-11-14T14:52:13,419 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34445/hbase/data/default/testReplayEditsWrittenViaHRegion/b525c5f69b8d59a438788e12849dfa00/recovered.edits/33.seqid, newMaxSeqId=33, maxSeqId=1 2024-11-14T14:52:13,420 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened b525c5f69b8d59a438788e12849dfa00; next sequenceid=34; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71898423, jitterRate=0.07136999070644379}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-14T14:52:13,421 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for b525c5f69b8d59a438788e12849dfa00: Writing region info on filesystem at 1731595933318Initializing all the Stores at 1731595933319 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731595933319Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731595933319Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731595933319Obtaining lock to block concurrent updates at 1731595933371 (+52 ms)Preparing flush snapshotting stores in b525c5f69b8d59a438788e12849dfa00 at 1731595933371Finished memstore snapshotting testReplayEditsWrittenViaHRegion,,1731595932883.b525c5f69b8d59a438788e12849dfa00., syncing WAL and waiting on mvcc, flushsize=dataSize=870, getHeapSize=2320, getOffHeapSize=0, getCellsCount=10 at 1731595933371Flushing stores of testReplayEditsWrittenViaHRegion,,1731595932883.b525c5f69b8d59a438788e12849dfa00. at 1731595933371Flushing b525c5f69b8d59a438788e12849dfa00/b: creating writer at 1731595933371Flushing b525c5f69b8d59a438788e12849dfa00/b: appending metadata at 1731595933393 (+22 ms)Flushing b525c5f69b8d59a438788e12849dfa00/b: closing flushed file at 1731595933393Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7494cc49: reopening flushed file at 1731595933408 (+15 ms)Finished flush of dataSize ~870 B/870, heapSize ~1.80 KB/1840, currentSize=0 B/0 for b525c5f69b8d59a438788e12849dfa00 in 43ms, sequenceid=32, compaction requested=false; wal=null at 1731595933414 (+6 ms)Cleaning up temporary data from old regions at 1731595933415 (+1 ms)Region opened successfully at 1731595933421 (+6 ms) 2024-11-14T14:52:13,447 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsAfterPartialFlush Thread=419 (was 410) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1791616411_22 at /127.0.0.1:33870 [Waiting for operation #32] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1055376130-172.17.0.2-1731595909704:blk_1073741899_1077, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1791616411_22 at /127.0.0.1:34744 [Receiving block BP-1055376130-172.17.0.2-1731595909704:blk_1073741899_1077] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1791616411_22 at /127.0.0.1:55178 [Receiving block BP-1055376130-172.17.0.2-1731595909704:blk_1073741899_1077] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-20-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-20-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1791616411_22 at /127.0.0.1:53360 [Waiting for operation #31] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1791616411_22 at /127.0.0.1:49250 [Receiving block BP-1055376130-172.17.0.2-1731595909704:blk_1073741899_1077] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1791616411_22 at /127.0.0.1:58010 [Waiting for operation #39] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1055376130-172.17.0.2-1731595909704:blk_1073741899_1077, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-20-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1055376130-172.17.0.2-1731595909704:blk_1073741899_1077, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1124 (was 1048) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=186 (was 186), ProcessCount=11 (was 11), AvailableMemoryMB=7581 (was 7581) 2024-11-14T14:52:13,447 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1124 is superior to 1024 2024-11-14T14:52:13,461 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsAfterAbortingFlush Thread=419, OpenFileDescriptor=1124, MaxFileDescriptor=1048576, SystemLoadAverage=186, ProcessCount=11, AvailableMemoryMB=7581 2024-11-14T14:52:13,461 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1124 is superior to 1024 2024-11-14T14:52:13,478 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-14T14:52:13,480 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-14T14:52:13,481 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-14T14:52:13,483 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-91516895, suffix=, logDir=hdfs://localhost:34445/hbase/WALs/hregion-91516895, archiveDir=hdfs://localhost:34445/hbase/oldWALs, maxLogs=32 2024-11-14T14:52:13,496 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-91516895/hregion-91516895.1731595933484, exclude list is [], retry=0 2024-11-14T14:52:13,499 DEBUG [AsyncFSWAL-22-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36189,DS-5c245456-9209-4937-bfad-43a9a1b09820,DISK] 2024-11-14T14:52:13,499 DEBUG [AsyncFSWAL-22-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44549,DS-913adb4a-7946-4f46-896c-56bcdc4fcfa8,DISK] 2024-11-14T14:52:13,500 DEBUG [AsyncFSWAL-22-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42819,DS-c1b8ccdc-5e2c-4b3d-bb1e-6941b40dfb7e,DISK] 2024-11-14T14:52:13,528 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-91516895/hregion-91516895.1731595933484 2024-11-14T14:52:13,528 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33623:33623),(127.0.0.1/127.0.0.1:33515:33515),(127.0.0.1/127.0.0.1:34777:34777)] 2024-11-14T14:52:13,529 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 0ff446145e3ceb7bee22636c8dc9f92a, NAME => 'testReplayEditsAfterAbortingFlush,,1731595933479.0ff446145e3ceb7bee22636c8dc9f92a.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsAfterAbortingFlush', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34445/hbase 2024-11-14T14:52:13,537 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42819 is added to blk_1073741902_1080 (size=68) 2024-11-14T14:52:13,537 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44549 is added to blk_1073741902_1080 (size=68) 2024-11-14T14:52:13,538 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36189 is added to blk_1073741902_1080 (size=68) 2024-11-14T14:52:13,538 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterAbortingFlush,,1731595933479.0ff446145e3ceb7bee22636c8dc9f92a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T14:52:13,540 INFO [StoreOpener-0ff446145e3ceb7bee22636c8dc9f92a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 0ff446145e3ceb7bee22636c8dc9f92a 2024-11-14T14:52:13,541 INFO [StoreOpener-0ff446145e3ceb7bee22636c8dc9f92a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0ff446145e3ceb7bee22636c8dc9f92a columnFamilyName a 2024-11-14T14:52:13,541 DEBUG [StoreOpener-0ff446145e3ceb7bee22636c8dc9f92a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T14:52:13,542 INFO [StoreOpener-0ff446145e3ceb7bee22636c8dc9f92a-1 {}] regionserver.HStore(327): Store=0ff446145e3ceb7bee22636c8dc9f92a/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T14:52:13,542 INFO [StoreOpener-0ff446145e3ceb7bee22636c8dc9f92a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 0ff446145e3ceb7bee22636c8dc9f92a 2024-11-14T14:52:13,543 INFO [StoreOpener-0ff446145e3ceb7bee22636c8dc9f92a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0ff446145e3ceb7bee22636c8dc9f92a columnFamilyName b 2024-11-14T14:52:13,543 DEBUG [StoreOpener-0ff446145e3ceb7bee22636c8dc9f92a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T14:52:13,544 INFO [StoreOpener-0ff446145e3ceb7bee22636c8dc9f92a-1 {}] regionserver.HStore(327): Store=0ff446145e3ceb7bee22636c8dc9f92a/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T14:52:13,544 INFO [StoreOpener-0ff446145e3ceb7bee22636c8dc9f92a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 0ff446145e3ceb7bee22636c8dc9f92a 2024-11-14T14:52:13,545 INFO [StoreOpener-0ff446145e3ceb7bee22636c8dc9f92a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0ff446145e3ceb7bee22636c8dc9f92a columnFamilyName c 2024-11-14T14:52:13,545 DEBUG [StoreOpener-0ff446145e3ceb7bee22636c8dc9f92a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T14:52:13,545 INFO [StoreOpener-0ff446145e3ceb7bee22636c8dc9f92a-1 {}] regionserver.HStore(327): Store=0ff446145e3ceb7bee22636c8dc9f92a/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T14:52:13,546 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 0ff446145e3ceb7bee22636c8dc9f92a 2024-11-14T14:52:13,546 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34445/hbase/data/default/testReplayEditsAfterAbortingFlush/0ff446145e3ceb7bee22636c8dc9f92a 2024-11-14T14:52:13,547 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34445/hbase/data/default/testReplayEditsAfterAbortingFlush/0ff446145e3ceb7bee22636c8dc9f92a 2024-11-14T14:52:13,548 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 0ff446145e3ceb7bee22636c8dc9f92a 2024-11-14T14:52:13,548 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 0ff446145e3ceb7bee22636c8dc9f92a 2024-11-14T14:52:13,548 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterAbortingFlush descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-14T14:52:13,550 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 0ff446145e3ceb7bee22636c8dc9f92a 2024-11-14T14:52:13,552 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34445/hbase/data/default/testReplayEditsAfterAbortingFlush/0ff446145e3ceb7bee22636c8dc9f92a/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-14T14:52:13,552 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 0ff446145e3ceb7bee22636c8dc9f92a; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68088744, jitterRate=0.014601349830627441}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-14T14:52:13,553 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 0ff446145e3ceb7bee22636c8dc9f92a: Writing region info on filesystem at 1731595933539Initializing all the Stores at 1731595933539Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731595933539Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731595933540 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731595933540Cleaning up temporary data from old regions at 1731595933548 (+8 ms)Region opened successfully at 1731595933553 (+5 ms) 2024-11-14T14:52:13,553 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 0ff446145e3ceb7bee22636c8dc9f92a, disabling compactions & flushes 2024-11-14T14:52:13,553 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterAbortingFlush,,1731595933479.0ff446145e3ceb7bee22636c8dc9f92a. 2024-11-14T14:52:13,553 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterAbortingFlush,,1731595933479.0ff446145e3ceb7bee22636c8dc9f92a. 2024-11-14T14:52:13,553 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterAbortingFlush,,1731595933479.0ff446145e3ceb7bee22636c8dc9f92a. after waiting 0 ms 2024-11-14T14:52:13,553 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterAbortingFlush,,1731595933479.0ff446145e3ceb7bee22636c8dc9f92a. 2024-11-14T14:52:13,554 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsAfterAbortingFlush,,1731595933479.0ff446145e3ceb7bee22636c8dc9f92a. 2024-11-14T14:52:13,554 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 0ff446145e3ceb7bee22636c8dc9f92a: Waiting for close lock at 1731595933553Disabling compacts and flushes for region at 1731595933553Disabling writes for close at 1731595933553Writing region close event to WAL at 1731595933554 (+1 ms)Closed at 1731595933554 2024-11-14T14:52:13,557 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42819 is added to blk_1073741901_1079 (size=175) 2024-11-14T14:52:13,557 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36189 is added to blk_1073741901_1079 (size=175) 2024-11-14T14:52:13,557 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44549 is added to blk_1073741901_1079 (size=175) 2024-11-14T14:52:13,559 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-14T14:52:13,560 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-91516895:(num 1731595933484) 2024-11-14T14:52:13,560 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-14T14:52:13,562 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:34445/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731595933478, archiveDir=hdfs://localhost:34445/hbase/oldWALs, maxLogs=32 2024-11-14T14:52:13,576 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731595933478/wal.1731595933562, exclude list is [], retry=0 2024-11-14T14:52:13,579 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44549,DS-913adb4a-7946-4f46-896c-56bcdc4fcfa8,DISK] 2024-11-14T14:52:13,579 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42819,DS-c1b8ccdc-5e2c-4b3d-bb1e-6941b40dfb7e,DISK] 2024-11-14T14:52:13,579 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36189,DS-5c245456-9209-4937-bfad-43a9a1b09820,DISK] 2024-11-14T14:52:13,605 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731595933478/wal.1731595933562 2024-11-14T14:52:13,606 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33515:33515),(127.0.0.1/127.0.0.1:34777:34777),(127.0.0.1/127.0.0.1:33623:33623)] 2024-11-14T14:52:13,660 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 0ff446145e3ceb7bee22636c8dc9f92a, NAME => 'testReplayEditsAfterAbortingFlush,,1731595933479.0ff446145e3ceb7bee22636c8dc9f92a.', STARTKEY => '', ENDKEY => ''} 2024-11-14T14:52:13,663 DEBUG [Time-limited test {}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterAbortingFlush 0ff446145e3ceb7bee22636c8dc9f92a 2024-11-14T14:52:13,663 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterAbortingFlush,,1731595933479.0ff446145e3ceb7bee22636c8dc9f92a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T14:52:13,663 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 0ff446145e3ceb7bee22636c8dc9f92a 2024-11-14T14:52:13,663 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 0ff446145e3ceb7bee22636c8dc9f92a 2024-11-14T14:52:13,665 INFO [StoreOpener-0ff446145e3ceb7bee22636c8dc9f92a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 0ff446145e3ceb7bee22636c8dc9f92a 2024-11-14T14:52:13,666 INFO [StoreOpener-0ff446145e3ceb7bee22636c8dc9f92a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0ff446145e3ceb7bee22636c8dc9f92a columnFamilyName a 2024-11-14T14:52:13,666 DEBUG [StoreOpener-0ff446145e3ceb7bee22636c8dc9f92a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T14:52:13,667 INFO [StoreOpener-0ff446145e3ceb7bee22636c8dc9f92a-1 {}] regionserver.HStore(327): Store=0ff446145e3ceb7bee22636c8dc9f92a/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T14:52:13,667 INFO [StoreOpener-0ff446145e3ceb7bee22636c8dc9f92a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 0ff446145e3ceb7bee22636c8dc9f92a 2024-11-14T14:52:13,668 INFO [StoreOpener-0ff446145e3ceb7bee22636c8dc9f92a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0ff446145e3ceb7bee22636c8dc9f92a columnFamilyName b 2024-11-14T14:52:13,668 DEBUG [StoreOpener-0ff446145e3ceb7bee22636c8dc9f92a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T14:52:13,668 INFO [StoreOpener-0ff446145e3ceb7bee22636c8dc9f92a-1 {}] regionserver.HStore(327): Store=0ff446145e3ceb7bee22636c8dc9f92a/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T14:52:13,668 INFO [StoreOpener-0ff446145e3ceb7bee22636c8dc9f92a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 0ff446145e3ceb7bee22636c8dc9f92a 2024-11-14T14:52:13,669 INFO [StoreOpener-0ff446145e3ceb7bee22636c8dc9f92a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0ff446145e3ceb7bee22636c8dc9f92a columnFamilyName c 2024-11-14T14:52:13,669 DEBUG [StoreOpener-0ff446145e3ceb7bee22636c8dc9f92a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T14:52:13,670 INFO [StoreOpener-0ff446145e3ceb7bee22636c8dc9f92a-1 {}] regionserver.HStore(327): Store=0ff446145e3ceb7bee22636c8dc9f92a/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T14:52:13,670 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 0ff446145e3ceb7bee22636c8dc9f92a 2024-11-14T14:52:13,670 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34445/hbase/data/default/testReplayEditsAfterAbortingFlush/0ff446145e3ceb7bee22636c8dc9f92a 2024-11-14T14:52:13,672 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34445/hbase/data/default/testReplayEditsAfterAbortingFlush/0ff446145e3ceb7bee22636c8dc9f92a 2024-11-14T14:52:13,672 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 0ff446145e3ceb7bee22636c8dc9f92a 2024-11-14T14:52:13,672 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 0ff446145e3ceb7bee22636c8dc9f92a 2024-11-14T14:52:13,673 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterAbortingFlush descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-14T14:52:13,674 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 0ff446145e3ceb7bee22636c8dc9f92a 2024-11-14T14:52:13,675 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 0ff446145e3ceb7bee22636c8dc9f92a; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65661511, jitterRate=-0.021567240357398987}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-14T14:52:13,675 DEBUG [Time-limited test {}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 0ff446145e3ceb7bee22636c8dc9f92a 2024-11-14T14:52:13,676 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 0ff446145e3ceb7bee22636c8dc9f92a: Running coprocessor pre-open hook at 1731595933663Writing region info on filesystem at 1731595933663Initializing all the Stores at 1731595933664 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731595933664Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731595933665 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731595933665Cleaning up temporary data from old regions at 1731595933672 (+7 ms)Running coprocessor post-open hooks at 1731595933675 (+3 ms)Region opened successfully at 1731595933676 (+1 ms) 2024-11-14T14:52:13,688 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 0ff446145e3ceb7bee22636c8dc9f92a 3/3 column families, dataSize=590 B heapSize=2.08 KB 2024-11-14T14:52:13,689 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 0ff446145e3ceb7bee22636c8dc9f92a/a, retrying num=0 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:52:14,689 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 0ff446145e3ceb7bee22636c8dc9f92a/a, retrying num=1 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:52:15,437 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-14T14:52:15,690 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 0ff446145e3ceb7bee22636c8dc9f92a/a, retrying num=2 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:52:16,691 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 0ff446145e3ceb7bee22636c8dc9f92a/a, retrying num=3 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:52:17,691 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 0ff446145e3ceb7bee22636c8dc9f92a/a, retrying num=4 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:52:18,692 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 0ff446145e3ceb7bee22636c8dc9f92a/a, retrying num=5 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:52:19,693 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 0ff446145e3ceb7bee22636c8dc9f92a/a, retrying num=6 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:52:20,693 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 0ff446145e3ceb7bee22636c8dc9f92a/a, retrying num=7 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:52:21,694 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 0ff446145e3ceb7bee22636c8dc9f92a/a, retrying num=8 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:52:21,883 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-14T14:52:22,694 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 0ff446145e3ceb7bee22636c8dc9f92a/a, retrying num=9 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:52:22,696 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 0ff446145e3ceb7bee22636c8dc9f92a: 2024-11-14T14:52:22,696 INFO [Time-limited test {}] wal.AbstractTestWALReplay(671): Expected simulated exception when flushing region, region: testReplayEditsAfterAbortingFlush,,1731595933479.0ff446145e3ceb7bee22636c8dc9f92a. 2024-11-14T14:52:22,713 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 0ff446145e3ceb7bee22636c8dc9f92a: 2024-11-14T14:52:22,713 INFO [Time-limited test {}] wal.AbstractTestWALReplay(691): Expected exception when flushing region because server is stopped,Aborting flush because server is aborted... 2024-11-14T14:52:22,714 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 0ff446145e3ceb7bee22636c8dc9f92a, disabling compactions & flushes 2024-11-14T14:52:22,714 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterAbortingFlush,,1731595933479.0ff446145e3ceb7bee22636c8dc9f92a. 2024-11-14T14:52:22,714 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterAbortingFlush,,1731595933479.0ff446145e3ceb7bee22636c8dc9f92a. 2024-11-14T14:52:22,714 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterAbortingFlush,,1731595933479.0ff446145e3ceb7bee22636c8dc9f92a. after waiting 0 ms 2024-11-14T14:52:22,714 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterAbortingFlush,,1731595933479.0ff446145e3ceb7bee22636c8dc9f92a. 2024-11-14T14:52:22,714 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 1190 in region testReplayEditsAfterAbortingFlush,,1731595933479.0ff446145e3ceb7bee22636c8dc9f92a. 2024-11-14T14:52:22,714 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsAfterAbortingFlush,,1731595933479.0ff446145e3ceb7bee22636c8dc9f92a. 2024-11-14T14:52:22,714 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 0ff446145e3ceb7bee22636c8dc9f92a: Waiting for close lock at 1731595942714Running coprocessor pre-close hooks at 1731595942714Disabling compacts and flushes for region at 1731595942714Disabling writes for close at 1731595942714Writing region close event to WAL at 1731595942714Running coprocessor post-close hooks at 1731595942714Closed at 1731595942714 2024-11-14T14:52:22,717 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36189 is added to blk_1073741903_1081 (size=4236) 2024-11-14T14:52:22,717 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42819 is added to blk_1073741903_1081 (size=4236) 2024-11-14T14:52:22,717 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44549 is added to blk_1073741903_1081 (size=4236) 2024-11-14T14:52:22,733 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:34445/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731595933478/wal.1731595933562, size=4.1 K (4236bytes) 2024-11-14T14:52:22,733 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:34445/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731595933478/wal.1731595933562 2024-11-14T14:52:22,734 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:34445/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731595933478/wal.1731595933562 after 1ms 2024-11-14T14:52:22,761 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-14T14:52:22,761 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:34445/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731595933478/wal.1731595933562 took 28ms 2024-11-14T14:52:22,764 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:34445/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731595933478/wal.1731595933562 so closing down 2024-11-14T14:52:22,764 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-14T14:52:22,765 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-wal.1731595933562.temp 2024-11-14T14:52:22,790 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:34445/hbase/data/default/testReplayEditsAfterAbortingFlush/0ff446145e3ceb7bee22636c8dc9f92a/recovered.edits/0000000000000000004-wal.1731595933562.temp 2024-11-14T14:52:22,790 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-14T14:52:22,797 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36189 is added to blk_1073741904_1082 (size=3245) 2024-11-14T14:52:22,797 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42819 is added to blk_1073741904_1082 (size=3245) 2024-11-14T14:52:22,797 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44549 is added to blk_1073741904_1082 (size=3245) 2024-11-14T14:52:22,798 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:34445/hbase/data/default/testReplayEditsAfterAbortingFlush/0ff446145e3ceb7bee22636c8dc9f92a/recovered.edits/0000000000000000004-wal.1731595933562.temp (wrote 20 edits, skipped 0 edits in 0 ms) 2024-11-14T14:52:22,799 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:34445/hbase/data/default/testReplayEditsAfterAbortingFlush/0ff446145e3ceb7bee22636c8dc9f92a/recovered.edits/0000000000000000004-wal.1731595933562.temp to hdfs://localhost:34445/hbase/data/default/testReplayEditsAfterAbortingFlush/0ff446145e3ceb7bee22636c8dc9f92a/recovered.edits/0000000000000000026 2024-11-14T14:52:22,799 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 23 edits across 1 Regions in 37 ms; skipped=3; WAL=hdfs://localhost:34445/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731595933478/wal.1731595933562, size=4.1 K, length=4236, corrupted=false, cancelled=false 2024-11-14T14:52:22,799 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:34445/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731595933478/wal.1731595933562, journal: Splitting hdfs://localhost:34445/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731595933478/wal.1731595933562, size=4.1 K (4236bytes) at 1731595942733Finishing writing output for hdfs://localhost:34445/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731595933478/wal.1731595933562 so closing down at 1731595942764 (+31 ms)Creating recovered edits writer path=hdfs://localhost:34445/hbase/data/default/testReplayEditsAfterAbortingFlush/0ff446145e3ceb7bee22636c8dc9f92a/recovered.edits/0000000000000000004-wal.1731595933562.temp at 1731595942790 (+26 ms)3 split writer threads finished at 1731595942791 (+1 ms)Closed recovered edits writer path=hdfs://localhost:34445/hbase/data/default/testReplayEditsAfterAbortingFlush/0ff446145e3ceb7bee22636c8dc9f92a/recovered.edits/0000000000000000004-wal.1731595933562.temp (wrote 20 edits, skipped 0 edits in 0 ms) at 1731595942798 (+7 ms)Rename recovered edits hdfs://localhost:34445/hbase/data/default/testReplayEditsAfterAbortingFlush/0ff446145e3ceb7bee22636c8dc9f92a/recovered.edits/0000000000000000004-wal.1731595933562.temp to hdfs://localhost:34445/hbase/data/default/testReplayEditsAfterAbortingFlush/0ff446145e3ceb7bee22636c8dc9f92a/recovered.edits/0000000000000000026 at 1731595942799 (+1 ms)Processed 23 edits across 1 Regions in 37 ms; skipped=3; WAL=hdfs://localhost:34445/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731595933478/wal.1731595933562, size=4.1 K, length=4236, corrupted=false, cancelled=false at 1731595942799 2024-11-14T14:52:22,801 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:34445/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731595933478/wal.1731595933562 to hdfs://localhost:34445/hbase/oldWALs/wal.1731595933562 2024-11-14T14:52:22,802 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:34445/hbase/data/default/testReplayEditsAfterAbortingFlush/0ff446145e3ceb7bee22636c8dc9f92a/recovered.edits/0000000000000000026 2024-11-14T14:52:22,802 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-14T14:52:22,803 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:34445/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731595933478, archiveDir=hdfs://localhost:34445/hbase/oldWALs, maxLogs=32 2024-11-14T14:52:22,816 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731595933478/wal.1731595942804, exclude list is [], retry=0 2024-11-14T14:52:22,818 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44549,DS-913adb4a-7946-4f46-896c-56bcdc4fcfa8,DISK] 2024-11-14T14:52:22,819 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42819,DS-c1b8ccdc-5e2c-4b3d-bb1e-6941b40dfb7e,DISK] 2024-11-14T14:52:22,819 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36189,DS-5c245456-9209-4937-bfad-43a9a1b09820,DISK] 2024-11-14T14:52:22,845 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731595933478/wal.1731595942804 2024-11-14T14:52:22,846 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33515:33515),(127.0.0.1/127.0.0.1:34777:34777),(127.0.0.1/127.0.0.1:33623:33623)] 2024-11-14T14:52:22,846 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 0ff446145e3ceb7bee22636c8dc9f92a, NAME => 'testReplayEditsAfterAbortingFlush,,1731595933479.0ff446145e3ceb7bee22636c8dc9f92a.', STARTKEY => '', ENDKEY => ''} 2024-11-14T14:52:22,847 DEBUG [Time-limited test {}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterAbortingFlush 0ff446145e3ceb7bee22636c8dc9f92a 2024-11-14T14:52:22,847 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterAbortingFlush,,1731595933479.0ff446145e3ceb7bee22636c8dc9f92a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T14:52:22,847 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 0ff446145e3ceb7bee22636c8dc9f92a 2024-11-14T14:52:22,847 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 0ff446145e3ceb7bee22636c8dc9f92a 2024-11-14T14:52:22,849 INFO [StoreOpener-0ff446145e3ceb7bee22636c8dc9f92a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 0ff446145e3ceb7bee22636c8dc9f92a 2024-11-14T14:52:22,850 INFO [StoreOpener-0ff446145e3ceb7bee22636c8dc9f92a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0ff446145e3ceb7bee22636c8dc9f92a columnFamilyName a 2024-11-14T14:52:22,850 DEBUG [StoreOpener-0ff446145e3ceb7bee22636c8dc9f92a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T14:52:22,850 INFO [StoreOpener-0ff446145e3ceb7bee22636c8dc9f92a-1 {}] regionserver.HStore(327): Store=0ff446145e3ceb7bee22636c8dc9f92a/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T14:52:22,850 INFO [StoreOpener-0ff446145e3ceb7bee22636c8dc9f92a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 0ff446145e3ceb7bee22636c8dc9f92a 2024-11-14T14:52:22,851 INFO [StoreOpener-0ff446145e3ceb7bee22636c8dc9f92a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0ff446145e3ceb7bee22636c8dc9f92a columnFamilyName b 2024-11-14T14:52:22,851 DEBUG [StoreOpener-0ff446145e3ceb7bee22636c8dc9f92a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T14:52:22,852 INFO [StoreOpener-0ff446145e3ceb7bee22636c8dc9f92a-1 {}] regionserver.HStore(327): Store=0ff446145e3ceb7bee22636c8dc9f92a/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T14:52:22,852 INFO [StoreOpener-0ff446145e3ceb7bee22636c8dc9f92a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 0ff446145e3ceb7bee22636c8dc9f92a 2024-11-14T14:52:22,853 INFO [StoreOpener-0ff446145e3ceb7bee22636c8dc9f92a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0ff446145e3ceb7bee22636c8dc9f92a columnFamilyName c 2024-11-14T14:52:22,853 DEBUG [StoreOpener-0ff446145e3ceb7bee22636c8dc9f92a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T14:52:22,853 INFO [StoreOpener-0ff446145e3ceb7bee22636c8dc9f92a-1 {}] regionserver.HStore(327): Store=0ff446145e3ceb7bee22636c8dc9f92a/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T14:52:22,853 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 0ff446145e3ceb7bee22636c8dc9f92a 2024-11-14T14:52:22,854 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34445/hbase/data/default/testReplayEditsAfterAbortingFlush/0ff446145e3ceb7bee22636c8dc9f92a 2024-11-14T14:52:22,855 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:34445/hbase/data/default/testReplayEditsAfterAbortingFlush/0ff446145e3ceb7bee22636c8dc9f92a 2024-11-14T14:52:22,856 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:34445/hbase/data/default/testReplayEditsAfterAbortingFlush/0ff446145e3ceb7bee22636c8dc9f92a/recovered.edits/0000000000000000026 2024-11-14T14:52:22,884 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-14T14:52:22,886 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 20, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=26, path=hdfs://localhost:34445/hbase/data/default/testReplayEditsAfterAbortingFlush/0ff446145e3ceb7bee22636c8dc9f92a/recovered.edits/0000000000000000026 2024-11-14T14:52:22,886 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 0ff446145e3ceb7bee22636c8dc9f92a 3/3 column families, dataSize=1.16 KB heapSize=3.41 KB 2024-11-14T14:52:22,902 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/hbase/data/default/testReplayEditsAfterAbortingFlush/0ff446145e3ceb7bee22636c8dc9f92a/.tmp/a/b8c4cece7b744302aebb8f0b1258b6bb is 64, key is testReplayEditsAfterAbortingFlush12/a:q/1731595942702/Put/seqid=0 2024-11-14T14:52:22,909 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42819 is added to blk_1073741906_1084 (size=5523) 2024-11-14T14:52:22,910 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36189 is added to blk_1073741906_1084 (size=5523) 2024-11-14T14:52:22,910 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44549 is added to blk_1073741906_1084 (size=5523) 2024-11-14T14:52:22,910 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=416 B at sequenceid=26 (bloomFilter=true), to=hdfs://localhost:34445/hbase/data/default/testReplayEditsAfterAbortingFlush/0ff446145e3ceb7bee22636c8dc9f92a/.tmp/a/b8c4cece7b744302aebb8f0b1258b6bb 2024-11-14T14:52:22,931 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/hbase/data/default/testReplayEditsAfterAbortingFlush/0ff446145e3ceb7bee22636c8dc9f92a/.tmp/b/10a8bba8a44947b1954f41dda5c8f57e is 64, key is testReplayEditsAfterAbortingFlush10/b:q/1731595942696/Put/seqid=0 2024-11-14T14:52:22,938 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44549 is added to blk_1073741907_1085 (size=5524) 2024-11-14T14:52:22,938 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36189 is added to blk_1073741907_1085 (size=5524) 2024-11-14T14:52:22,939 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42819 is added to blk_1073741907_1085 (size=5524) 2024-11-14T14:52:22,940 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=417 B at sequenceid=26 (bloomFilter=true), to=hdfs://localhost:34445/hbase/data/default/testReplayEditsAfterAbortingFlush/0ff446145e3ceb7bee22636c8dc9f92a/.tmp/b/10a8bba8a44947b1954f41dda5c8f57e 2024-11-14T14:52:22,960 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/hbase/data/default/testReplayEditsAfterAbortingFlush/0ff446145e3ceb7bee22636c8dc9f92a/.tmp/c/b5bcc807324a47a3985b39d41caa61f0 is 64, key is testReplayEditsAfterAbortingFlush11/c:q/1731595942699/Put/seqid=0 2024-11-14T14:52:22,967 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44549 is added to blk_1073741908_1086 (size=5457) 2024-11-14T14:52:22,967 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36189 is added to blk_1073741908_1086 (size=5457) 2024-11-14T14:52:22,968 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42819 is added to blk_1073741908_1086 (size=5457) 2024-11-14T14:52:22,968 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=357 B at sequenceid=26 (bloomFilter=true), to=hdfs://localhost:34445/hbase/data/default/testReplayEditsAfterAbortingFlush/0ff446145e3ceb7bee22636c8dc9f92a/.tmp/c/b5bcc807324a47a3985b39d41caa61f0 2024-11-14T14:52:22,974 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/hbase/data/default/testReplayEditsAfterAbortingFlush/0ff446145e3ceb7bee22636c8dc9f92a/.tmp/a/b8c4cece7b744302aebb8f0b1258b6bb as hdfs://localhost:34445/hbase/data/default/testReplayEditsAfterAbortingFlush/0ff446145e3ceb7bee22636c8dc9f92a/a/b8c4cece7b744302aebb8f0b1258b6bb 2024-11-14T14:52:22,980 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/hbase/data/default/testReplayEditsAfterAbortingFlush/0ff446145e3ceb7bee22636c8dc9f92a/a/b8c4cece7b744302aebb8f0b1258b6bb, entries=7, sequenceid=26, filesize=5.4 K 2024-11-14T14:52:22,981 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/hbase/data/default/testReplayEditsAfterAbortingFlush/0ff446145e3ceb7bee22636c8dc9f92a/.tmp/b/10a8bba8a44947b1954f41dda5c8f57e as hdfs://localhost:34445/hbase/data/default/testReplayEditsAfterAbortingFlush/0ff446145e3ceb7bee22636c8dc9f92a/b/10a8bba8a44947b1954f41dda5c8f57e 2024-11-14T14:52:22,987 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/hbase/data/default/testReplayEditsAfterAbortingFlush/0ff446145e3ceb7bee22636c8dc9f92a/b/10a8bba8a44947b1954f41dda5c8f57e, entries=7, sequenceid=26, filesize=5.4 K 2024-11-14T14:52:22,988 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/hbase/data/default/testReplayEditsAfterAbortingFlush/0ff446145e3ceb7bee22636c8dc9f92a/.tmp/c/b5bcc807324a47a3985b39d41caa61f0 as hdfs://localhost:34445/hbase/data/default/testReplayEditsAfterAbortingFlush/0ff446145e3ceb7bee22636c8dc9f92a/c/b5bcc807324a47a3985b39d41caa61f0 2024-11-14T14:52:22,993 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/hbase/data/default/testReplayEditsAfterAbortingFlush/0ff446145e3ceb7bee22636c8dc9f92a/c/b5bcc807324a47a3985b39d41caa61f0, entries=6, sequenceid=26, filesize=5.3 K 2024-11-14T14:52:22,993 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~1.16 KB/1190, heapSize ~3.36 KB/3440, currentSize=0 B/0 for 0ff446145e3ceb7bee22636c8dc9f92a in 107ms, sequenceid=26, compaction requested=false; wal=null 2024-11-14T14:52:22,994 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:34445/hbase/data/default/testReplayEditsAfterAbortingFlush/0ff446145e3ceb7bee22636c8dc9f92a/recovered.edits/0000000000000000026 2024-11-14T14:52:22,995 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 0ff446145e3ceb7bee22636c8dc9f92a 2024-11-14T14:52:22,995 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 0ff446145e3ceb7bee22636c8dc9f92a 2024-11-14T14:52:22,996 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterAbortingFlush descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-14T14:52:22,998 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 0ff446145e3ceb7bee22636c8dc9f92a 2024-11-14T14:52:23,000 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34445/hbase/data/default/testReplayEditsAfterAbortingFlush/0ff446145e3ceb7bee22636c8dc9f92a/recovered.edits/26.seqid, newMaxSeqId=26, maxSeqId=1 2024-11-14T14:52:23,001 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 0ff446145e3ceb7bee22636c8dc9f92a; next sequenceid=27; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60517764, jitterRate=-0.09821504354476929}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-14T14:52:23,001 DEBUG [Time-limited test {}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 0ff446145e3ceb7bee22636c8dc9f92a 2024-11-14T14:52:23,002 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 0ff446145e3ceb7bee22636c8dc9f92a: Running coprocessor pre-open hook at 1731595942847Writing region info on filesystem at 1731595942847Initializing all the Stores at 1731595942848 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731595942848Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731595942848Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731595942848Obtaining lock to block concurrent updates at 1731595942886 (+38 ms)Preparing flush snapshotting stores in 0ff446145e3ceb7bee22636c8dc9f92a at 1731595942886Finished memstore snapshotting testReplayEditsAfterAbortingFlush,,1731595933479.0ff446145e3ceb7bee22636c8dc9f92a., syncing WAL and waiting on mvcc, flushsize=dataSize=1190, getHeapSize=3440, getOffHeapSize=0, getCellsCount=20 at 1731595942886Flushing stores of testReplayEditsAfterAbortingFlush,,1731595933479.0ff446145e3ceb7bee22636c8dc9f92a. at 1731595942886Flushing 0ff446145e3ceb7bee22636c8dc9f92a/a: creating writer at 1731595942887 (+1 ms)Flushing 0ff446145e3ceb7bee22636c8dc9f92a/a: appending metadata at 1731595942901 (+14 ms)Flushing 0ff446145e3ceb7bee22636c8dc9f92a/a: closing flushed file at 1731595942902 (+1 ms)Flushing 0ff446145e3ceb7bee22636c8dc9f92a/b: creating writer at 1731595942916 (+14 ms)Flushing 0ff446145e3ceb7bee22636c8dc9f92a/b: appending metadata at 1731595942930 (+14 ms)Flushing 0ff446145e3ceb7bee22636c8dc9f92a/b: closing flushed file at 1731595942930Flushing 0ff446145e3ceb7bee22636c8dc9f92a/c: creating writer at 1731595942945 (+15 ms)Flushing 0ff446145e3ceb7bee22636c8dc9f92a/c: appending metadata at 1731595942960 (+15 ms)Flushing 0ff446145e3ceb7bee22636c8dc9f92a/c: closing flushed file at 1731595942960Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5c53cbd6: reopening flushed file at 1731595942973 (+13 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@cd592b4: reopening flushed file at 1731595942980 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@435a07cb: reopening flushed file at 1731595942987 (+7 ms)Finished flush of dataSize ~1.16 KB/1190, heapSize ~3.36 KB/3440, currentSize=0 B/0 for 0ff446145e3ceb7bee22636c8dc9f92a in 107ms, sequenceid=26, compaction requested=false; wal=null at 1731595942993 (+6 ms)Cleaning up temporary data from old regions at 1731595942995 (+2 ms)Running coprocessor post-open hooks at 1731595943001 (+6 ms)Region opened successfully at 1731595943001 2024-11-14T14:52:23,023 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsAfterAbortingFlush Thread=419 (was 419), OpenFileDescriptor=1182 (was 1124) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=158 (was 186), ProcessCount=11 (was 11), AvailableMemoryMB=7558 (was 7581) 2024-11-14T14:52:23,023 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1182 is superior to 1024 2024-11-14T14:52:23,036 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testDatalossWhenInputError Thread=419, OpenFileDescriptor=1182, MaxFileDescriptor=1048576, SystemLoadAverage=158, ProcessCount=11, AvailableMemoryMB=7557 2024-11-14T14:52:23,036 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1182 is superior to 1024 2024-11-14T14:52:23,051 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-14T14:52:23,076 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-14T14:52:23,077 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-14T14:52:23,079 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-45388446, suffix=, logDir=hdfs://localhost:34445/hbase/WALs/hregion-45388446, archiveDir=hdfs://localhost:34445/hbase/oldWALs, maxLogs=32 2024-11-14T14:52:23,093 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-45388446/hregion-45388446.1731595943080, exclude list is [], retry=0 2024-11-14T14:52:23,096 DEBUG [AsyncFSWAL-24-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44549,DS-913adb4a-7946-4f46-896c-56bcdc4fcfa8,DISK] 2024-11-14T14:52:23,096 DEBUG [AsyncFSWAL-24-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42819,DS-c1b8ccdc-5e2c-4b3d-bb1e-6941b40dfb7e,DISK] 2024-11-14T14:52:23,096 DEBUG [AsyncFSWAL-24-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36189,DS-5c245456-9209-4937-bfad-43a9a1b09820,DISK] 2024-11-14T14:52:23,128 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-45388446/hregion-45388446.1731595943080 2024-11-14T14:52:23,129 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33515:33515),(127.0.0.1/127.0.0.1:34777:34777),(127.0.0.1/127.0.0.1:33623:33623)] 2024-11-14T14:52:23,129 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 9351f2cfbfadc4ab65209517cefc3ba9, NAME => 'testDatalossWhenInputError,,1731595943051.9351f2cfbfadc4ab65209517cefc3ba9.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testDatalossWhenInputError', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34445/hbase 2024-11-14T14:52:23,138 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42819 is added to blk_1073741910_1088 (size=61) 2024-11-14T14:52:23,138 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44549 is added to blk_1073741910_1088 (size=61) 2024-11-14T14:52:23,139 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36189 is added to blk_1073741910_1088 (size=61) 2024-11-14T14:52:23,139 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1731595943051.9351f2cfbfadc4ab65209517cefc3ba9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T14:52:23,140 INFO [StoreOpener-9351f2cfbfadc4ab65209517cefc3ba9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 9351f2cfbfadc4ab65209517cefc3ba9 2024-11-14T14:52:23,142 INFO [StoreOpener-9351f2cfbfadc4ab65209517cefc3ba9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9351f2cfbfadc4ab65209517cefc3ba9 columnFamilyName a 2024-11-14T14:52:23,142 DEBUG [StoreOpener-9351f2cfbfadc4ab65209517cefc3ba9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T14:52:23,143 INFO [StoreOpener-9351f2cfbfadc4ab65209517cefc3ba9-1 {}] regionserver.HStore(327): Store=9351f2cfbfadc4ab65209517cefc3ba9/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T14:52:23,143 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 9351f2cfbfadc4ab65209517cefc3ba9 2024-11-14T14:52:23,144 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34445/hbase/data/default/testDatalossWhenInputError/9351f2cfbfadc4ab65209517cefc3ba9 2024-11-14T14:52:23,144 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34445/hbase/data/default/testDatalossWhenInputError/9351f2cfbfadc4ab65209517cefc3ba9 2024-11-14T14:52:23,144 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 9351f2cfbfadc4ab65209517cefc3ba9 2024-11-14T14:52:23,144 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 9351f2cfbfadc4ab65209517cefc3ba9 2024-11-14T14:52:23,146 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 9351f2cfbfadc4ab65209517cefc3ba9 2024-11-14T14:52:23,148 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34445/hbase/data/default/testDatalossWhenInputError/9351f2cfbfadc4ab65209517cefc3ba9/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-14T14:52:23,149 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 9351f2cfbfadc4ab65209517cefc3ba9; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63472639, jitterRate=-0.054183974862098694}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-14T14:52:23,149 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 9351f2cfbfadc4ab65209517cefc3ba9: Writing region info on filesystem at 1731595943139Initializing all the Stores at 1731595943140 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731595943140Cleaning up temporary data from old regions at 1731595943144 (+4 ms)Region opened successfully at 1731595943149 (+5 ms) 2024-11-14T14:52:23,149 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 9351f2cfbfadc4ab65209517cefc3ba9, disabling compactions & flushes 2024-11-14T14:52:23,149 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testDatalossWhenInputError,,1731595943051.9351f2cfbfadc4ab65209517cefc3ba9. 2024-11-14T14:52:23,149 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testDatalossWhenInputError,,1731595943051.9351f2cfbfadc4ab65209517cefc3ba9. 2024-11-14T14:52:23,149 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testDatalossWhenInputError,,1731595943051.9351f2cfbfadc4ab65209517cefc3ba9. after waiting 0 ms 2024-11-14T14:52:23,149 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testDatalossWhenInputError,,1731595943051.9351f2cfbfadc4ab65209517cefc3ba9. 2024-11-14T14:52:23,150 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testDatalossWhenInputError,,1731595943051.9351f2cfbfadc4ab65209517cefc3ba9. 2024-11-14T14:52:23,150 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 9351f2cfbfadc4ab65209517cefc3ba9: Waiting for close lock at 1731595943149Disabling compacts and flushes for region at 1731595943149Disabling writes for close at 1731595943149Writing region close event to WAL at 1731595943150 (+1 ms)Closed at 1731595943150 2024-11-14T14:52:23,152 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36189 is added to blk_1073741909_1087 (size=175) 2024-11-14T14:52:23,152 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterAbortingFlush 2024-11-14T14:52:23,153 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterAbortingFlush Metrics about Tables on a single HBase RegionServer 2024-11-14T14:52:23,153 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42819 is added to blk_1073741909_1087 (size=175) 2024-11-14T14:52:23,153 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44549 is added to blk_1073741909_1087 (size=175) 2024-11-14T14:52:23,157 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-14T14:52:23,157 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-45388446:(num 1731595943080) 2024-11-14T14:52:23,157 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-14T14:52:23,159 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:34445/hbase/WALs/testdatalosswheninputerror-manual,16010,1731595943050, archiveDir=hdfs://localhost:34445/hbase/oldWALs, maxLogs=32 2024-11-14T14:52:23,172 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testdatalosswheninputerror-manual,16010,1731595943050/wal.1731595943159, exclude list is [], retry=0 2024-11-14T14:52:23,175 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44549,DS-913adb4a-7946-4f46-896c-56bcdc4fcfa8,DISK] 2024-11-14T14:52:23,176 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36189,DS-5c245456-9209-4937-bfad-43a9a1b09820,DISK] 2024-11-14T14:52:23,176 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42819,DS-c1b8ccdc-5e2c-4b3d-bb1e-6941b40dfb7e,DISK] 2024-11-14T14:52:23,204 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testdatalosswheninputerror-manual,16010,1731595943050/wal.1731595943159 2024-11-14T14:52:23,204 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33515:33515),(127.0.0.1/127.0.0.1:33623:33623),(127.0.0.1/127.0.0.1:34777:34777)] 2024-11-14T14:52:23,205 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 9351f2cfbfadc4ab65209517cefc3ba9, NAME => 'testDatalossWhenInputError,,1731595943051.9351f2cfbfadc4ab65209517cefc3ba9.', STARTKEY => '', ENDKEY => ''} 2024-11-14T14:52:23,205 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1731595943051.9351f2cfbfadc4ab65209517cefc3ba9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T14:52:23,205 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 9351f2cfbfadc4ab65209517cefc3ba9 2024-11-14T14:52:23,205 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 9351f2cfbfadc4ab65209517cefc3ba9 2024-11-14T14:52:23,207 INFO [StoreOpener-9351f2cfbfadc4ab65209517cefc3ba9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 9351f2cfbfadc4ab65209517cefc3ba9 2024-11-14T14:52:23,208 INFO [StoreOpener-9351f2cfbfadc4ab65209517cefc3ba9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9351f2cfbfadc4ab65209517cefc3ba9 columnFamilyName a 2024-11-14T14:52:23,208 DEBUG [StoreOpener-9351f2cfbfadc4ab65209517cefc3ba9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T14:52:23,208 INFO [StoreOpener-9351f2cfbfadc4ab65209517cefc3ba9-1 {}] regionserver.HStore(327): Store=9351f2cfbfadc4ab65209517cefc3ba9/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T14:52:23,208 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 9351f2cfbfadc4ab65209517cefc3ba9 2024-11-14T14:52:23,209 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34445/hbase/data/default/testDatalossWhenInputError/9351f2cfbfadc4ab65209517cefc3ba9 2024-11-14T14:52:23,211 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34445/hbase/data/default/testDatalossWhenInputError/9351f2cfbfadc4ab65209517cefc3ba9 2024-11-14T14:52:23,211 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 9351f2cfbfadc4ab65209517cefc3ba9 2024-11-14T14:52:23,211 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 9351f2cfbfadc4ab65209517cefc3ba9 2024-11-14T14:52:23,214 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 9351f2cfbfadc4ab65209517cefc3ba9 2024-11-14T14:52:23,215 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 9351f2cfbfadc4ab65209517cefc3ba9; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66998867, jitterRate=-0.0016390830278396606}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-14T14:52:23,216 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 9351f2cfbfadc4ab65209517cefc3ba9: Writing region info on filesystem at 1731595943205Initializing all the Stores at 1731595943206 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731595943206Cleaning up temporary data from old regions at 1731595943211 (+5 ms)Region opened successfully at 1731595943216 (+5 ms) 2024-11-14T14:52:23,228 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 9351f2cfbfadc4ab65209517cefc3ba9, disabling compactions & flushes 2024-11-14T14:52:23,228 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testDatalossWhenInputError,,1731595943051.9351f2cfbfadc4ab65209517cefc3ba9. 2024-11-14T14:52:23,228 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testDatalossWhenInputError,,1731595943051.9351f2cfbfadc4ab65209517cefc3ba9. 2024-11-14T14:52:23,228 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testDatalossWhenInputError,,1731595943051.9351f2cfbfadc4ab65209517cefc3ba9. after waiting 0 ms 2024-11-14T14:52:23,228 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testDatalossWhenInputError,,1731595943051.9351f2cfbfadc4ab65209517cefc3ba9. 2024-11-14T14:52:23,229 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 750 in region testDatalossWhenInputError,,1731595943051.9351f2cfbfadc4ab65209517cefc3ba9. 2024-11-14T14:52:23,229 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testDatalossWhenInputError,,1731595943051.9351f2cfbfadc4ab65209517cefc3ba9. 2024-11-14T14:52:23,229 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 9351f2cfbfadc4ab65209517cefc3ba9: Waiting for close lock at 1731595943228Disabling compacts and flushes for region at 1731595943228Disabling writes for close at 1731595943228Writing region close event to WAL at 1731595943229 (+1 ms)Closed at 1731595943229 2024-11-14T14:52:23,232 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42819 is added to blk_1073741911_1089 (size=1795) 2024-11-14T14:52:23,232 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36189 is added to blk_1073741911_1089 (size=1795) 2024-11-14T14:52:23,232 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44549 is added to blk_1073741911_1089 (size=1795) 2024-11-14T14:52:23,249 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:34445/hbase/WALs/testdatalosswheninputerror-manual,16010,1731595943050/wal.1731595943159, size=1.8 K (1795bytes) 2024-11-14T14:52:23,249 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:34445/hbase/WALs/testdatalosswheninputerror-manual,16010,1731595943050/wal.1731595943159 2024-11-14T14:52:23,250 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:34445/hbase/WALs/testdatalosswheninputerror-manual,16010,1731595943050/wal.1731595943159 after 1ms 2024-11-14T14:52:23,279 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-14T14:52:23,279 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:34445/hbase/WALs/testdatalosswheninputerror-manual,16010,1731595943050/wal.1731595943159 took 30ms 2024-11-14T14:52:23,281 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:34445/hbase/WALs/testdatalosswheninputerror-manual,16010,1731595943050/wal.1731595943159 so closing down 2024-11-14T14:52:23,282 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-14T14:52:23,283 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1731595943159.temp 2024-11-14T14:52:23,320 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:34445/hbase/data/default/testDatalossWhenInputError/9351f2cfbfadc4ab65209517cefc3ba9/recovered.edits/0000000000000000003-wal.1731595943159.temp 2024-11-14T14:52:23,320 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-14T14:52:23,328 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42819 is added to blk_1073741912_1090 (size=1795) 2024-11-14T14:52:23,328 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44549 is added to blk_1073741912_1090 (size=1795) 2024-11-14T14:52:23,329 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36189 is added to blk_1073741912_1090 (size=1795) 2024-11-14T14:52:23,329 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:34445/hbase/data/default/testDatalossWhenInputError/9351f2cfbfadc4ab65209517cefc3ba9/recovered.edits/0000000000000000003-wal.1731595943159.temp (wrote 10 edits, skipped 0 edits in 0 ms) 2024-11-14T14:52:23,331 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:34445/hbase/data/default/testDatalossWhenInputError/9351f2cfbfadc4ab65209517cefc3ba9/recovered.edits/0000000000000000003-wal.1731595943159.temp to hdfs://localhost:34445/hbase/data/default/testDatalossWhenInputError/9351f2cfbfadc4ab65209517cefc3ba9/recovered.edits/0000000000000000012 2024-11-14T14:52:23,331 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 10 edits across 1 Regions in 51 ms; skipped=0; WAL=hdfs://localhost:34445/hbase/WALs/testdatalosswheninputerror-manual,16010,1731595943050/wal.1731595943159, size=1.8 K, length=1795, corrupted=false, cancelled=false 2024-11-14T14:52:23,331 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:34445/hbase/WALs/testdatalosswheninputerror-manual,16010,1731595943050/wal.1731595943159, journal: Splitting hdfs://localhost:34445/hbase/WALs/testdatalosswheninputerror-manual,16010,1731595943050/wal.1731595943159, size=1.8 K (1795bytes) at 1731595943249Finishing writing output for hdfs://localhost:34445/hbase/WALs/testdatalosswheninputerror-manual,16010,1731595943050/wal.1731595943159 so closing down at 1731595943282 (+33 ms)Creating recovered edits writer path=hdfs://localhost:34445/hbase/data/default/testDatalossWhenInputError/9351f2cfbfadc4ab65209517cefc3ba9/recovered.edits/0000000000000000003-wal.1731595943159.temp at 1731595943320 (+38 ms)3 split writer threads finished at 1731595943320Closed recovered edits writer path=hdfs://localhost:34445/hbase/data/default/testDatalossWhenInputError/9351f2cfbfadc4ab65209517cefc3ba9/recovered.edits/0000000000000000003-wal.1731595943159.temp (wrote 10 edits, skipped 0 edits in 0 ms) at 1731595943329 (+9 ms)Rename recovered edits hdfs://localhost:34445/hbase/data/default/testDatalossWhenInputError/9351f2cfbfadc4ab65209517cefc3ba9/recovered.edits/0000000000000000003-wal.1731595943159.temp to hdfs://localhost:34445/hbase/data/default/testDatalossWhenInputError/9351f2cfbfadc4ab65209517cefc3ba9/recovered.edits/0000000000000000012 at 1731595943331 (+2 ms)Processed 10 edits across 1 Regions in 51 ms; skipped=0; WAL=hdfs://localhost:34445/hbase/WALs/testdatalosswheninputerror-manual,16010,1731595943050/wal.1731595943159, size=1.8 K, length=1795, corrupted=false, cancelled=false at 1731595943331 2024-11-14T14:52:23,333 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:34445/hbase/WALs/testdatalosswheninputerror-manual,16010,1731595943050/wal.1731595943159 to hdfs://localhost:34445/hbase/oldWALs/wal.1731595943159 2024-11-14T14:52:23,334 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:34445/hbase/data/default/testDatalossWhenInputError/9351f2cfbfadc4ab65209517cefc3ba9/recovered.edits/0000000000000000012 2024-11-14T14:52:23,364 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-14T14:52:23,732 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-14T14:52:23,736 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:34445/hbase/WALs/testdatalosswheninputerror-manual,16010,1731595943050, archiveDir=hdfs://localhost:34445/hbase/oldWALs, maxLogs=32 2024-11-14T14:52:23,752 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testdatalosswheninputerror-manual,16010,1731595943050/wal.1731595943737, exclude list is [], retry=0 2024-11-14T14:52:23,755 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36189,DS-5c245456-9209-4937-bfad-43a9a1b09820,DISK] 2024-11-14T14:52:23,755 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42819,DS-c1b8ccdc-5e2c-4b3d-bb1e-6941b40dfb7e,DISK] 2024-11-14T14:52:23,755 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44549,DS-913adb4a-7946-4f46-896c-56bcdc4fcfa8,DISK] 2024-11-14T14:52:23,798 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testdatalosswheninputerror-manual,16010,1731595943050/wal.1731595943737 2024-11-14T14:52:23,798 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33623:33623),(127.0.0.1/127.0.0.1:34777:34777),(127.0.0.1/127.0.0.1:33515:33515)] 2024-11-14T14:52:23,799 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 9351f2cfbfadc4ab65209517cefc3ba9, NAME => 'testDatalossWhenInputError,,1731595943051.9351f2cfbfadc4ab65209517cefc3ba9.', STARTKEY => '', ENDKEY => ''} 2024-11-14T14:52:23,799 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1731595943051.9351f2cfbfadc4ab65209517cefc3ba9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T14:52:23,799 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 9351f2cfbfadc4ab65209517cefc3ba9 2024-11-14T14:52:23,799 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 9351f2cfbfadc4ab65209517cefc3ba9 2024-11-14T14:52:23,803 INFO [StoreOpener-9351f2cfbfadc4ab65209517cefc3ba9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 9351f2cfbfadc4ab65209517cefc3ba9 2024-11-14T14:52:23,806 INFO [StoreOpener-9351f2cfbfadc4ab65209517cefc3ba9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9351f2cfbfadc4ab65209517cefc3ba9 columnFamilyName a 2024-11-14T14:52:23,806 DEBUG [StoreOpener-9351f2cfbfadc4ab65209517cefc3ba9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T14:52:23,807 INFO [StoreOpener-9351f2cfbfadc4ab65209517cefc3ba9-1 {}] regionserver.HStore(327): Store=9351f2cfbfadc4ab65209517cefc3ba9/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T14:52:23,807 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 9351f2cfbfadc4ab65209517cefc3ba9 2024-11-14T14:52:23,808 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34445/hbase/data/default/testDatalossWhenInputError/9351f2cfbfadc4ab65209517cefc3ba9 2024-11-14T14:52:23,810 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:34445/hbase/data/default/testDatalossWhenInputError/9351f2cfbfadc4ab65209517cefc3ba9 2024-11-14T14:52:23,811 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:34445/hbase/data/default/testDatalossWhenInputError/9351f2cfbfadc4ab65209517cefc3ba9/recovered.edits/0000000000000000012 2024-11-14T14:52:23,855 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-14T14:52:23,857 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 10, skipped 0, firstSequenceIdInLog=3, maxSequenceIdInLog=12, path=hdfs://localhost:34445/hbase/data/default/testDatalossWhenInputError/9351f2cfbfadc4ab65209517cefc3ba9/recovered.edits/0000000000000000012 2024-11-14T14:52:23,857 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 9351f2cfbfadc4ab65209517cefc3ba9 1/1 column families, dataSize=750 B heapSize=1.73 KB 2024-11-14T14:52:23,878 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/hbase/data/default/testDatalossWhenInputError/9351f2cfbfadc4ab65209517cefc3ba9/.tmp/a/616606e5be0641139636d41789d45769 is 79, key is testDatalossWhenInputError/a:x0/1731595943216/Put/seqid=0 2024-11-14T14:52:23,885 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44549 is added to blk_1073741914_1092 (size=5808) 2024-11-14T14:52:23,885 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42819 is added to blk_1073741914_1092 (size=5808) 2024-11-14T14:52:23,886 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36189 is added to blk_1073741914_1092 (size=5808) 2024-11-14T14:52:23,887 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=750 B at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:34445/hbase/data/default/testDatalossWhenInputError/9351f2cfbfadc4ab65209517cefc3ba9/.tmp/a/616606e5be0641139636d41789d45769 2024-11-14T14:52:23,897 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/hbase/data/default/testDatalossWhenInputError/9351f2cfbfadc4ab65209517cefc3ba9/.tmp/a/616606e5be0641139636d41789d45769 as hdfs://localhost:34445/hbase/data/default/testDatalossWhenInputError/9351f2cfbfadc4ab65209517cefc3ba9/a/616606e5be0641139636d41789d45769 2024-11-14T14:52:23,907 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/hbase/data/default/testDatalossWhenInputError/9351f2cfbfadc4ab65209517cefc3ba9/a/616606e5be0641139636d41789d45769, entries=10, sequenceid=12, filesize=5.7 K 2024-11-14T14:52:23,907 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~750 B/750, heapSize ~1.72 KB/1760, currentSize=0 B/0 for 9351f2cfbfadc4ab65209517cefc3ba9 in 50ms, sequenceid=12, compaction requested=false; wal=null 2024-11-14T14:52:23,908 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:34445/hbase/data/default/testDatalossWhenInputError/9351f2cfbfadc4ab65209517cefc3ba9/recovered.edits/0000000000000000012 2024-11-14T14:52:23,909 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 9351f2cfbfadc4ab65209517cefc3ba9 2024-11-14T14:52:23,909 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 9351f2cfbfadc4ab65209517cefc3ba9 2024-11-14T14:52:23,913 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 9351f2cfbfadc4ab65209517cefc3ba9 2024-11-14T14:52:23,915 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34445/hbase/data/default/testDatalossWhenInputError/9351f2cfbfadc4ab65209517cefc3ba9/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=1 2024-11-14T14:52:23,916 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 9351f2cfbfadc4ab65209517cefc3ba9; next sequenceid=13; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62585253, jitterRate=-0.06740705668926239}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-14T14:52:23,917 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 9351f2cfbfadc4ab65209517cefc3ba9: Writing region info on filesystem at 1731595943799Initializing all the Stores at 1731595943803 (+4 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731595943803Obtaining lock to block concurrent updates at 1731595943857 (+54 ms)Preparing flush snapshotting stores in 9351f2cfbfadc4ab65209517cefc3ba9 at 1731595943857Finished memstore snapshotting testDatalossWhenInputError,,1731595943051.9351f2cfbfadc4ab65209517cefc3ba9., syncing WAL and waiting on mvcc, flushsize=dataSize=750, getHeapSize=1760, getOffHeapSize=0, getCellsCount=10 at 1731595943857Flushing stores of testDatalossWhenInputError,,1731595943051.9351f2cfbfadc4ab65209517cefc3ba9. at 1731595943857Flushing 9351f2cfbfadc4ab65209517cefc3ba9/a: creating writer at 1731595943857Flushing 9351f2cfbfadc4ab65209517cefc3ba9/a: appending metadata at 1731595943877 (+20 ms)Flushing 9351f2cfbfadc4ab65209517cefc3ba9/a: closing flushed file at 1731595943877Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3f62f7e3: reopening flushed file at 1731595943895 (+18 ms)Finished flush of dataSize ~750 B/750, heapSize ~1.72 KB/1760, currentSize=0 B/0 for 9351f2cfbfadc4ab65209517cefc3ba9 in 50ms, sequenceid=12, compaction requested=false; wal=null at 1731595943907 (+12 ms)Cleaning up temporary data from old regions at 1731595943909 (+2 ms)Region opened successfully at 1731595943916 (+7 ms) 2024-11-14T14:52:23,920 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 9351f2cfbfadc4ab65209517cefc3ba9, NAME => 'testDatalossWhenInputError,,1731595943051.9351f2cfbfadc4ab65209517cefc3ba9.', STARTKEY => '', ENDKEY => ''} 2024-11-14T14:52:23,920 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1731595943051.9351f2cfbfadc4ab65209517cefc3ba9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T14:52:23,920 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 9351f2cfbfadc4ab65209517cefc3ba9 2024-11-14T14:52:23,920 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 9351f2cfbfadc4ab65209517cefc3ba9 2024-11-14T14:52:23,922 INFO [StoreOpener-9351f2cfbfadc4ab65209517cefc3ba9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 9351f2cfbfadc4ab65209517cefc3ba9 2024-11-14T14:52:23,923 INFO [StoreOpener-9351f2cfbfadc4ab65209517cefc3ba9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9351f2cfbfadc4ab65209517cefc3ba9 columnFamilyName a 2024-11-14T14:52:23,923 DEBUG [StoreOpener-9351f2cfbfadc4ab65209517cefc3ba9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T14:52:23,929 DEBUG [StoreOpener-9351f2cfbfadc4ab65209517cefc3ba9-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:34445/hbase/data/default/testDatalossWhenInputError/9351f2cfbfadc4ab65209517cefc3ba9/a/616606e5be0641139636d41789d45769 2024-11-14T14:52:23,929 INFO [StoreOpener-9351f2cfbfadc4ab65209517cefc3ba9-1 {}] regionserver.HStore(327): Store=9351f2cfbfadc4ab65209517cefc3ba9/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T14:52:23,930 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 9351f2cfbfadc4ab65209517cefc3ba9 2024-11-14T14:52:23,931 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34445/hbase/data/default/testDatalossWhenInputError/9351f2cfbfadc4ab65209517cefc3ba9 2024-11-14T14:52:23,934 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34445/hbase/data/default/testDatalossWhenInputError/9351f2cfbfadc4ab65209517cefc3ba9 2024-11-14T14:52:23,934 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 9351f2cfbfadc4ab65209517cefc3ba9 2024-11-14T14:52:23,934 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 9351f2cfbfadc4ab65209517cefc3ba9 2024-11-14T14:52:23,936 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 9351f2cfbfadc4ab65209517cefc3ba9 2024-11-14T14:52:23,939 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34445/hbase/data/default/testDatalossWhenInputError/9351f2cfbfadc4ab65209517cefc3ba9/recovered.edits/13.seqid, newMaxSeqId=13, maxSeqId=12 2024-11-14T14:52:23,940 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 9351f2cfbfadc4ab65209517cefc3ba9; next sequenceid=14; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75474775, jitterRate=0.12466178834438324}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-14T14:52:23,940 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 9351f2cfbfadc4ab65209517cefc3ba9: Writing region info on filesystem at 1731595943920Initializing all the Stores at 1731595943921 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731595943921Cleaning up temporary data from old regions at 1731595943934 (+13 ms)Region opened successfully at 1731595943940 (+6 ms) 2024-11-14T14:52:23,959 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testDatalossWhenInputError Thread=429 (was 419) Potentially hanging thread: PacketResponder: BP-1055376130-172.17.0.2-1731595909704:blk_1073741913_1091, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1791616411_22 at /127.0.0.1:36828 [Receiving block BP-1055376130-172.17.0.2-1731595909704:blk_1073741913_1091] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-24-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-24-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1791616411_22 at /127.0.0.1:42918 [Waiting for operation #16] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1791616411_22 at /127.0.0.1:36766 [Waiting for operation #12] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1791616411_22 at /127.0.0.1:43004 [Receiving block BP-1055376130-172.17.0.2-1731595909704:blk_1073741913_1091] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1791616411_22 at /127.0.0.1:51762 [Receiving block BP-1055376130-172.17.0.2-1731595909704:blk_1073741913_1091] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1055376130-172.17.0.2-1731595909704:blk_1073741913_1091, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-24-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1055376130-172.17.0.2-1731595909704:blk_1073741913_1091, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1791616411_22 at /127.0.0.1:51670 [Waiting for operation #14] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1264 (was 1182) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=158 (was 158), ProcessCount=11 (was 11), AvailableMemoryMB=7524 (was 7557) 2024-11-14T14:52:23,959 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1264 is superior to 1024 2024-11-14T14:52:23,973 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testCompactedBulkLoadedFiles Thread=429, OpenFileDescriptor=1264, MaxFileDescriptor=1048576, SystemLoadAverage=158, ProcessCount=11, AvailableMemoryMB=7522 2024-11-14T14:52:23,973 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1264 is superior to 1024 2024-11-14T14:52:23,988 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-14T14:52:23,990 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-14T14:52:23,990 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-14T14:52:23,993 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-20128259, suffix=, logDir=hdfs://localhost:34445/hbase/WALs/hregion-20128259, archiveDir=hdfs://localhost:34445/hbase/oldWALs, maxLogs=32 2024-11-14T14:52:24,006 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-20128259/hregion-20128259.1731595943993, exclude list is [], retry=0 2024-11-14T14:52:24,009 DEBUG [AsyncFSWAL-26-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42819,DS-c1b8ccdc-5e2c-4b3d-bb1e-6941b40dfb7e,DISK] 2024-11-14T14:52:24,009 DEBUG [AsyncFSWAL-26-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36189,DS-5c245456-9209-4937-bfad-43a9a1b09820,DISK] 2024-11-14T14:52:24,009 DEBUG [AsyncFSWAL-26-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44549,DS-913adb4a-7946-4f46-896c-56bcdc4fcfa8,DISK] 2024-11-14T14:52:24,043 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-20128259/hregion-20128259.1731595943993 2024-11-14T14:52:24,043 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34777:34777),(127.0.0.1/127.0.0.1:33623:33623),(127.0.0.1/127.0.0.1:33515:33515)] 2024-11-14T14:52:24,043 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => b14640b65a291b227cae93d513b0700e, NAME => 'testCompactedBulkLoadedFiles,,1731595943988.b14640b65a291b227cae93d513b0700e.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testCompactedBulkLoadedFiles', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34445/hbase 2024-11-14T14:52:24,054 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42819 is added to blk_1073741916_1094 (size=63) 2024-11-14T14:52:24,054 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44549 is added to blk_1073741916_1094 (size=63) 2024-11-14T14:52:24,055 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36189 is added to blk_1073741916_1094 (size=63) 2024-11-14T14:52:24,055 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testCompactedBulkLoadedFiles,,1731595943988.b14640b65a291b227cae93d513b0700e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T14:52:24,057 INFO [StoreOpener-b14640b65a291b227cae93d513b0700e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region b14640b65a291b227cae93d513b0700e 2024-11-14T14:52:24,058 INFO [StoreOpener-b14640b65a291b227cae93d513b0700e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b14640b65a291b227cae93d513b0700e columnFamilyName a 2024-11-14T14:52:24,058 DEBUG [StoreOpener-b14640b65a291b227cae93d513b0700e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T14:52:24,059 INFO [StoreOpener-b14640b65a291b227cae93d513b0700e-1 {}] regionserver.HStore(327): Store=b14640b65a291b227cae93d513b0700e/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T14:52:24,059 INFO [StoreOpener-b14640b65a291b227cae93d513b0700e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region b14640b65a291b227cae93d513b0700e 2024-11-14T14:52:24,060 INFO [StoreOpener-b14640b65a291b227cae93d513b0700e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b14640b65a291b227cae93d513b0700e columnFamilyName b 2024-11-14T14:52:24,060 DEBUG [StoreOpener-b14640b65a291b227cae93d513b0700e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T14:52:24,061 INFO [StoreOpener-b14640b65a291b227cae93d513b0700e-1 {}] regionserver.HStore(327): Store=b14640b65a291b227cae93d513b0700e/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T14:52:24,061 INFO [StoreOpener-b14640b65a291b227cae93d513b0700e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region b14640b65a291b227cae93d513b0700e 2024-11-14T14:52:24,062 INFO [StoreOpener-b14640b65a291b227cae93d513b0700e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b14640b65a291b227cae93d513b0700e columnFamilyName c 2024-11-14T14:52:24,062 DEBUG [StoreOpener-b14640b65a291b227cae93d513b0700e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T14:52:24,063 INFO [StoreOpener-b14640b65a291b227cae93d513b0700e-1 {}] regionserver.HStore(327): Store=b14640b65a291b227cae93d513b0700e/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T14:52:24,063 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for b14640b65a291b227cae93d513b0700e 2024-11-14T14:52:24,064 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34445/hbase/data/default/testCompactedBulkLoadedFiles/b14640b65a291b227cae93d513b0700e 2024-11-14T14:52:24,064 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34445/hbase/data/default/testCompactedBulkLoadedFiles/b14640b65a291b227cae93d513b0700e 2024-11-14T14:52:24,065 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for b14640b65a291b227cae93d513b0700e 2024-11-14T14:52:24,065 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for b14640b65a291b227cae93d513b0700e 2024-11-14T14:52:24,066 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testCompactedBulkLoadedFiles descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-14T14:52:24,067 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for b14640b65a291b227cae93d513b0700e 2024-11-14T14:52:24,069 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34445/hbase/data/default/testCompactedBulkLoadedFiles/b14640b65a291b227cae93d513b0700e/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-14T14:52:24,070 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened b14640b65a291b227cae93d513b0700e; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64070448, jitterRate=-0.04527592658996582}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-14T14:52:24,070 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for b14640b65a291b227cae93d513b0700e: Writing region info on filesystem at 1731595944055Initializing all the Stores at 1731595944056 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731595944056Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731595944056Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731595944056Cleaning up temporary data from old regions at 1731595944065 (+9 ms)Region opened successfully at 1731595944070 (+5 ms) 2024-11-14T14:52:24,071 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing b14640b65a291b227cae93d513b0700e, disabling compactions & flushes 2024-11-14T14:52:24,071 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testCompactedBulkLoadedFiles,,1731595943988.b14640b65a291b227cae93d513b0700e. 2024-11-14T14:52:24,071 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testCompactedBulkLoadedFiles,,1731595943988.b14640b65a291b227cae93d513b0700e. 2024-11-14T14:52:24,071 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testCompactedBulkLoadedFiles,,1731595943988.b14640b65a291b227cae93d513b0700e. after waiting 0 ms 2024-11-14T14:52:24,071 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testCompactedBulkLoadedFiles,,1731595943988.b14640b65a291b227cae93d513b0700e. 2024-11-14T14:52:24,071 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testCompactedBulkLoadedFiles,,1731595943988.b14640b65a291b227cae93d513b0700e. 2024-11-14T14:52:24,071 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for b14640b65a291b227cae93d513b0700e: Waiting for close lock at 1731595944071Disabling compacts and flushes for region at 1731595944071Disabling writes for close at 1731595944071Writing region close event to WAL at 1731595944071Closed at 1731595944071 2024-11-14T14:52:24,074 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36189 is added to blk_1073741915_1093 (size=175) 2024-11-14T14:52:24,074 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44549 is added to blk_1073741915_1093 (size=175) 2024-11-14T14:52:24,074 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42819 is added to blk_1073741915_1093 (size=175) 2024-11-14T14:52:24,076 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-14T14:52:24,076 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-20128259:(num 1731595943993) 2024-11-14T14:52:24,076 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-14T14:52:24,078 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:34445/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731595943987, archiveDir=hdfs://localhost:34445/hbase/oldWALs, maxLogs=32 2024-11-14T14:52:24,092 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731595943987/wal.1731595944079, exclude list is [], retry=0 2024-11-14T14:52:24,095 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42819,DS-c1b8ccdc-5e2c-4b3d-bb1e-6941b40dfb7e,DISK] 2024-11-14T14:52:24,096 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44549,DS-913adb4a-7946-4f46-896c-56bcdc4fcfa8,DISK] 2024-11-14T14:52:24,096 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36189,DS-5c245456-9209-4937-bfad-43a9a1b09820,DISK] 2024-11-14T14:52:24,124 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731595943987/wal.1731595944079 2024-11-14T14:52:24,124 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34777:34777),(127.0.0.1/127.0.0.1:33515:33515),(127.0.0.1/127.0.0.1:33623:33623)] 2024-11-14T14:52:24,124 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => b14640b65a291b227cae93d513b0700e, NAME => 'testCompactedBulkLoadedFiles,,1731595943988.b14640b65a291b227cae93d513b0700e.', STARTKEY => '', ENDKEY => ''} 2024-11-14T14:52:24,124 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testCompactedBulkLoadedFiles,,1731595943988.b14640b65a291b227cae93d513b0700e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T14:52:24,125 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for b14640b65a291b227cae93d513b0700e 2024-11-14T14:52:24,125 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for b14640b65a291b227cae93d513b0700e 2024-11-14T14:52:24,126 INFO [StoreOpener-b14640b65a291b227cae93d513b0700e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region b14640b65a291b227cae93d513b0700e 2024-11-14T14:52:24,127 INFO [StoreOpener-b14640b65a291b227cae93d513b0700e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b14640b65a291b227cae93d513b0700e columnFamilyName a 2024-11-14T14:52:24,127 DEBUG [StoreOpener-b14640b65a291b227cae93d513b0700e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T14:52:24,128 INFO [StoreOpener-b14640b65a291b227cae93d513b0700e-1 {}] regionserver.HStore(327): Store=b14640b65a291b227cae93d513b0700e/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T14:52:24,128 INFO [StoreOpener-b14640b65a291b227cae93d513b0700e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region b14640b65a291b227cae93d513b0700e 2024-11-14T14:52:24,129 INFO [StoreOpener-b14640b65a291b227cae93d513b0700e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b14640b65a291b227cae93d513b0700e columnFamilyName b 2024-11-14T14:52:24,129 DEBUG [StoreOpener-b14640b65a291b227cae93d513b0700e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T14:52:24,129 INFO [StoreOpener-b14640b65a291b227cae93d513b0700e-1 {}] regionserver.HStore(327): Store=b14640b65a291b227cae93d513b0700e/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T14:52:24,129 INFO [StoreOpener-b14640b65a291b227cae93d513b0700e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region b14640b65a291b227cae93d513b0700e 2024-11-14T14:52:24,130 INFO [StoreOpener-b14640b65a291b227cae93d513b0700e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b14640b65a291b227cae93d513b0700e columnFamilyName c 2024-11-14T14:52:24,130 DEBUG [StoreOpener-b14640b65a291b227cae93d513b0700e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T14:52:24,130 INFO [StoreOpener-b14640b65a291b227cae93d513b0700e-1 {}] regionserver.HStore(327): Store=b14640b65a291b227cae93d513b0700e/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T14:52:24,131 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for b14640b65a291b227cae93d513b0700e 2024-11-14T14:52:24,131 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34445/hbase/data/default/testCompactedBulkLoadedFiles/b14640b65a291b227cae93d513b0700e 2024-11-14T14:52:24,133 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34445/hbase/data/default/testCompactedBulkLoadedFiles/b14640b65a291b227cae93d513b0700e 2024-11-14T14:52:24,134 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for b14640b65a291b227cae93d513b0700e 2024-11-14T14:52:24,134 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for b14640b65a291b227cae93d513b0700e 2024-11-14T14:52:24,134 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testCompactedBulkLoadedFiles descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-14T14:52:24,136 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for b14640b65a291b227cae93d513b0700e 2024-11-14T14:52:24,137 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened b14640b65a291b227cae93d513b0700e; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70887983, jitterRate=0.05631326138973236}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-14T14:52:24,137 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for b14640b65a291b227cae93d513b0700e: Writing region info on filesystem at 1731595944125Initializing all the Stores at 1731595944126 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731595944126Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731595944126Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731595944126Cleaning up temporary data from old regions at 1731595944134 (+8 ms)Region opened successfully at 1731595944137 (+3 ms) 2024-11-14T14:52:24,142 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/hbase/testCompactedBulkLoadedFiles/hfile0 is 32, key is 000/a:a/1731595944142/Put/seqid=0 2024-11-14T14:52:24,151 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42819 is added to blk_1073741918_1096 (size=4875) 2024-11-14T14:52:24,153 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36189 is added to blk_1073741918_1096 (size=4875) 2024-11-14T14:52:24,153 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44549 is added to blk_1073741918_1096 (size=4875) 2024-11-14T14:52:24,157 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/hbase/testCompactedBulkLoadedFiles/hfile1 is 32, key is 100/a:a/1731595944156/Put/seqid=0 2024-11-14T14:52:24,167 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36189 is added to blk_1073741919_1097 (size=4875) 2024-11-14T14:52:24,168 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42819 is added to blk_1073741919_1097 (size=4875) 2024-11-14T14:52:24,168 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44549 is added to blk_1073741919_1097 (size=4875) 2024-11-14T14:52:24,170 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/hbase/testCompactedBulkLoadedFiles/hfile2 is 32, key is 200/a:a/1731595944170/Put/seqid=0 2024-11-14T14:52:24,176 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44549 is added to blk_1073741920_1098 (size=4875) 2024-11-14T14:52:24,176 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36189 is added to blk_1073741920_1098 (size=4875) 2024-11-14T14:52:24,177 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42819 is added to blk_1073741920_1098 (size=4875) 2024-11-14T14:52:24,177 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:34445/hbase/testCompactedBulkLoadedFiles/hfile0 for inclusion in b14640b65a291b227cae93d513b0700e/a 2024-11-14T14:52:24,181 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first=000 last=050 2024-11-14T14:52:24,181 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-11-14T14:52:24,181 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:34445/hbase/testCompactedBulkLoadedFiles/hfile1 for inclusion in b14640b65a291b227cae93d513b0700e/a 2024-11-14T14:52:24,185 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first=100 last=150 2024-11-14T14:52:24,185 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-11-14T14:52:24,185 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:34445/hbase/testCompactedBulkLoadedFiles/hfile2 for inclusion in b14640b65a291b227cae93d513b0700e/a 2024-11-14T14:52:24,188 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first=200 last=250 2024-11-14T14:52:24,188 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-11-14T14:52:24,188 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing b14640b65a291b227cae93d513b0700e 3/3 column families, dataSize=51 B heapSize=896 B 2024-11-14T14:52:24,203 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/hbase/data/default/testCompactedBulkLoadedFiles/b14640b65a291b227cae93d513b0700e/.tmp/a/12b9b2d9ff0e42c5a3d9981d6d08110c is 55, key is testCompactedBulkLoadedFiles/a:a/1731595944138/Put/seqid=0 2024-11-14T14:52:24,210 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36189 is added to blk_1073741921_1099 (size=5107) 2024-11-14T14:52:24,210 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42819 is added to blk_1073741921_1099 (size=5107) 2024-11-14T14:52:24,211 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44549 is added to blk_1073741921_1099 (size=5107) 2024-11-14T14:52:24,212 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51 B at sequenceid=4 (bloomFilter=true), to=hdfs://localhost:34445/hbase/data/default/testCompactedBulkLoadedFiles/b14640b65a291b227cae93d513b0700e/.tmp/a/12b9b2d9ff0e42c5a3d9981d6d08110c 2024-11-14T14:52:24,218 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/hbase/data/default/testCompactedBulkLoadedFiles/b14640b65a291b227cae93d513b0700e/.tmp/a/12b9b2d9ff0e42c5a3d9981d6d08110c as hdfs://localhost:34445/hbase/data/default/testCompactedBulkLoadedFiles/b14640b65a291b227cae93d513b0700e/a/12b9b2d9ff0e42c5a3d9981d6d08110c 2024-11-14T14:52:24,223 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/hbase/data/default/testCompactedBulkLoadedFiles/b14640b65a291b227cae93d513b0700e/a/12b9b2d9ff0e42c5a3d9981d6d08110c, entries=1, sequenceid=4, filesize=5.0 K 2024-11-14T14:52:24,224 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~51 B/51, heapSize ~368 B/368, currentSize=0 B/0 for b14640b65a291b227cae93d513b0700e in 36ms, sequenceid=4, compaction requested=false 2024-11-14T14:52:24,224 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for b14640b65a291b227cae93d513b0700e: 2024-11-14T14:52:24,226 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/hbase/testCompactedBulkLoadedFiles/hfile0 as hdfs://localhost:34445/hbase/data/default/testCompactedBulkLoadedFiles/b14640b65a291b227cae93d513b0700e/a/c9493fe89e7d4f4299249a6dfc961b6f_SeqId_4_ 2024-11-14T14:52:24,227 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/hbase/testCompactedBulkLoadedFiles/hfile1 as hdfs://localhost:34445/hbase/data/default/testCompactedBulkLoadedFiles/b14640b65a291b227cae93d513b0700e/a/3ded0b7ab55742a4b7c2ab80d1274a27_SeqId_4_ 2024-11-14T14:52:24,228 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/hbase/testCompactedBulkLoadedFiles/hfile2 as hdfs://localhost:34445/hbase/data/default/testCompactedBulkLoadedFiles/b14640b65a291b227cae93d513b0700e/a/dc722b8c8301425996f7b1e6cbf6397c_SeqId_4_ 2024-11-14T14:52:24,229 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:34445/hbase/testCompactedBulkLoadedFiles/hfile0 into b14640b65a291b227cae93d513b0700e/a as hdfs://localhost:34445/hbase/data/default/testCompactedBulkLoadedFiles/b14640b65a291b227cae93d513b0700e/a/c9493fe89e7d4f4299249a6dfc961b6f_SeqId_4_ - updating store file list. 2024-11-14T14:52:24,233 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for c9493fe89e7d4f4299249a6dfc961b6f_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-14T14:52:24,233 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:34445/hbase/data/default/testCompactedBulkLoadedFiles/b14640b65a291b227cae93d513b0700e/a/c9493fe89e7d4f4299249a6dfc961b6f_SeqId_4_ into b14640b65a291b227cae93d513b0700e/a 2024-11-14T14:52:24,233 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:34445/hbase/testCompactedBulkLoadedFiles/hfile0 into b14640b65a291b227cae93d513b0700e/a (new location: hdfs://localhost:34445/hbase/data/default/testCompactedBulkLoadedFiles/b14640b65a291b227cae93d513b0700e/a/c9493fe89e7d4f4299249a6dfc961b6f_SeqId_4_) 2024-11-14T14:52:24,234 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:34445/hbase/testCompactedBulkLoadedFiles/hfile1 into b14640b65a291b227cae93d513b0700e/a as hdfs://localhost:34445/hbase/data/default/testCompactedBulkLoadedFiles/b14640b65a291b227cae93d513b0700e/a/3ded0b7ab55742a4b7c2ab80d1274a27_SeqId_4_ - updating store file list. 2024-11-14T14:52:24,238 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for 3ded0b7ab55742a4b7c2ab80d1274a27_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-14T14:52:24,238 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:34445/hbase/data/default/testCompactedBulkLoadedFiles/b14640b65a291b227cae93d513b0700e/a/3ded0b7ab55742a4b7c2ab80d1274a27_SeqId_4_ into b14640b65a291b227cae93d513b0700e/a 2024-11-14T14:52:24,238 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:34445/hbase/testCompactedBulkLoadedFiles/hfile1 into b14640b65a291b227cae93d513b0700e/a (new location: hdfs://localhost:34445/hbase/data/default/testCompactedBulkLoadedFiles/b14640b65a291b227cae93d513b0700e/a/3ded0b7ab55742a4b7c2ab80d1274a27_SeqId_4_) 2024-11-14T14:52:24,239 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:34445/hbase/testCompactedBulkLoadedFiles/hfile2 into b14640b65a291b227cae93d513b0700e/a as hdfs://localhost:34445/hbase/data/default/testCompactedBulkLoadedFiles/b14640b65a291b227cae93d513b0700e/a/dc722b8c8301425996f7b1e6cbf6397c_SeqId_4_ - updating store file list. 2024-11-14T14:52:24,244 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for dc722b8c8301425996f7b1e6cbf6397c_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-14T14:52:24,244 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:34445/hbase/data/default/testCompactedBulkLoadedFiles/b14640b65a291b227cae93d513b0700e/a/dc722b8c8301425996f7b1e6cbf6397c_SeqId_4_ into b14640b65a291b227cae93d513b0700e/a 2024-11-14T14:52:24,244 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:34445/hbase/testCompactedBulkLoadedFiles/hfile2 into b14640b65a291b227cae93d513b0700e/a (new location: hdfs://localhost:34445/hbase/data/default/testCompactedBulkLoadedFiles/b14640b65a291b227cae93d513b0700e/a/dc722b8c8301425996f7b1e6cbf6397c_SeqId_4_) 2024-11-14T14:52:24,252 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T14:52:24,253 DEBUG [Time-limited test {}] regionserver.HStore(1541): b14640b65a291b227cae93d513b0700e/a is initiating major compaction (all files) 2024-11-14T14:52:24,253 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of b14640b65a291b227cae93d513b0700e/a in testCompactedBulkLoadedFiles,,1731595943988.b14640b65a291b227cae93d513b0700e. 2024-11-14T14:52:24,253 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34445/hbase/data/default/testCompactedBulkLoadedFiles/b14640b65a291b227cae93d513b0700e/a/12b9b2d9ff0e42c5a3d9981d6d08110c, hdfs://localhost:34445/hbase/data/default/testCompactedBulkLoadedFiles/b14640b65a291b227cae93d513b0700e/a/c9493fe89e7d4f4299249a6dfc961b6f_SeqId_4_, hdfs://localhost:34445/hbase/data/default/testCompactedBulkLoadedFiles/b14640b65a291b227cae93d513b0700e/a/3ded0b7ab55742a4b7c2ab80d1274a27_SeqId_4_, hdfs://localhost:34445/hbase/data/default/testCompactedBulkLoadedFiles/b14640b65a291b227cae93d513b0700e/a/dc722b8c8301425996f7b1e6cbf6397c_SeqId_4_] into tmpdir=hdfs://localhost:34445/hbase/data/default/testCompactedBulkLoadedFiles/b14640b65a291b227cae93d513b0700e/.tmp, totalSize=19.3 K 2024-11-14T14:52:24,254 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 12b9b2d9ff0e42c5a3d9981d6d08110c, keycount=1, bloomtype=ROW, size=5.0 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=1731595944138 2024-11-14T14:52:24,254 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting c9493fe89e7d4f4299249a6dfc961b6f_SeqId_4_, keycount=10, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=-9223372036854775808 2024-11-14T14:52:24,254 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 3ded0b7ab55742a4b7c2ab80d1274a27_SeqId_4_, keycount=10, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=-9223372036854775808 2024-11-14T14:52:24,255 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting dc722b8c8301425996f7b1e6cbf6397c_SeqId_4_, keycount=10, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=-9223372036854775808 2024-11-14T14:52:24,267 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/hbase/data/default/testCompactedBulkLoadedFiles/b14640b65a291b227cae93d513b0700e/.tmp/a/8e021d58363449a7b4985dedd979cee3 is 55, key is testCompactedBulkLoadedFiles/a:a/1731595944138/Put/seqid=0 2024-11-14T14:52:24,276 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44549 is added to blk_1073741922_1100 (size=6154) 2024-11-14T14:52:24,276 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36189 is added to blk_1073741922_1100 (size=6154) 2024-11-14T14:52:24,276 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42819 is added to blk_1073741922_1100 (size=6154) 2024-11-14T14:52:24,283 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/hbase/data/default/testCompactedBulkLoadedFiles/b14640b65a291b227cae93d513b0700e/.tmp/a/8e021d58363449a7b4985dedd979cee3 as hdfs://localhost:34445/hbase/data/default/testCompactedBulkLoadedFiles/b14640b65a291b227cae93d513b0700e/a/8e021d58363449a7b4985dedd979cee3 2024-11-14T14:52:24,291 INFO [Time-limited test {}] regionserver.HStore(1337): Completed major compaction of 4 (all) file(s) in b14640b65a291b227cae93d513b0700e/a of b14640b65a291b227cae93d513b0700e into 8e021d58363449a7b4985dedd979cee3(size=6.0 K), total size for store is 6.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T14:52:24,291 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for b14640b65a291b227cae93d513b0700e: 2024-11-14T14:52:24,291 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 0 store files, 0 compacting, 0 eligible, 16 blocking 2024-11-14T14:52:24,292 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 0 store files, 0 compacting, 0 eligible, 16 blocking 2024-11-14T14:52:24,330 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:34445/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731595943987/wal.1731595944079, size=0 (0bytes) 2024-11-14T14:52:24,330 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:34445/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731595943987/wal.1731595944079 might be still open, length is 0 2024-11-14T14:52:24,330 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:34445/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731595943987/wal.1731595944079 2024-11-14T14:52:24,331 WARN [IPC Server handler 1 on default port 34445 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731595943987/wal.1731595944079 has not been closed. Lease recovery is in progress. RecoveryId = 1101 for block blk_1073741917_1095 2024-11-14T14:52:24,331 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:34445/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731595943987/wal.1731595944079 after 1ms 2024-11-14T14:52:24,689 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1791616411_22 at /127.0.0.1:36860 [Receiving block BP-1055376130-172.17.0.2-1731595909704:blk_1073741917_1095] {}] datanode.DataXceiver(331): 127.0.0.1:42819:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:36860 dst: /127.0.0.1:42819 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:42819 remote=/127.0.0.1:36860]. Total timeout mills is 60000, 59601 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:52:24,690 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1791616411_22 at /127.0.0.1:51796 [Receiving block BP-1055376130-172.17.0.2-1731595909704:blk_1073741917_1095] {}] datanode.DataXceiver(331): 127.0.0.1:36189:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:51796 dst: /127.0.0.1:36189 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:52:24,690 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1791616411_22 at /127.0.0.1:43042 [Receiving block BP-1055376130-172.17.0.2-1731595909704:blk_1073741917_1095] {}] datanode.DataXceiver(331): 127.0.0.1:44549:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:43042 dst: /127.0.0.1:44549 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:52:24,694 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42819 is added to blk_1073741917_1101 (size=1794) 2024-11-14T14:52:24,695 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44549 is added to blk_1073741917_1101 (size=1794) 2024-11-14T14:52:28,243 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-14T14:52:28,332 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:34445/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731595943987/wal.1731595944079 after 4002ms 2024-11-14T14:52:28,361 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-14T14:52:28,361 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:34445/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731595943987/wal.1731595944079 took 4031ms 2024-11-14T14:52:28,363 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:34445/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731595943987/wal.1731595944079; continuing. 2024-11-14T14:52:28,363 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:34445/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731595943987/wal.1731595944079 so closing down 2024-11-14T14:52:28,363 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-14T14:52:28,365 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1731595944079.temp 2024-11-14T14:52:28,391 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:34445/hbase/data/default/testCompactedBulkLoadedFiles/b14640b65a291b227cae93d513b0700e/recovered.edits/0000000000000000003-wal.1731595944079.temp 2024-11-14T14:52:28,391 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-14T14:52:28,397 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44549 is added to blk_1073741923_1102 (size=800) 2024-11-14T14:52:28,398 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36189 is added to blk_1073741923_1102 (size=800) 2024-11-14T14:52:28,398 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42819 is added to blk_1073741923_1102 (size=800) 2024-11-14T14:52:28,399 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:34445/hbase/data/default/testCompactedBulkLoadedFiles/b14640b65a291b227cae93d513b0700e/recovered.edits/0000000000000000003-wal.1731595944079.temp (wrote 2 edits, skipped 0 edits in 0 ms) 2024-11-14T14:52:28,400 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:34445/hbase/data/default/testCompactedBulkLoadedFiles/b14640b65a291b227cae93d513b0700e/recovered.edits/0000000000000000003-wal.1731595944079.temp to hdfs://localhost:34445/hbase/data/default/testCompactedBulkLoadedFiles/b14640b65a291b227cae93d513b0700e/recovered.edits/0000000000000000008 2024-11-14T14:52:28,400 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 5 edits across 1 Regions in 39 ms; skipped=3; WAL=hdfs://localhost:34445/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731595943987/wal.1731595944079, size=0, length=0, corrupted=false, cancelled=false 2024-11-14T14:52:28,400 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:34445/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731595943987/wal.1731595944079, journal: Splitting hdfs://localhost:34445/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731595943987/wal.1731595944079, size=0 (0bytes) at 1731595944330Finishing writing output for hdfs://localhost:34445/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731595943987/wal.1731595944079 so closing down at 1731595948363 (+4033 ms)Creating recovered edits writer path=hdfs://localhost:34445/hbase/data/default/testCompactedBulkLoadedFiles/b14640b65a291b227cae93d513b0700e/recovered.edits/0000000000000000003-wal.1731595944079.temp at 1731595948391 (+28 ms)3 split writer threads finished at 1731595948391Closed recovered edits writer path=hdfs://localhost:34445/hbase/data/default/testCompactedBulkLoadedFiles/b14640b65a291b227cae93d513b0700e/recovered.edits/0000000000000000003-wal.1731595944079.temp (wrote 2 edits, skipped 0 edits in 0 ms) at 1731595948399 (+8 ms)Rename recovered edits hdfs://localhost:34445/hbase/data/default/testCompactedBulkLoadedFiles/b14640b65a291b227cae93d513b0700e/recovered.edits/0000000000000000003-wal.1731595944079.temp to hdfs://localhost:34445/hbase/data/default/testCompactedBulkLoadedFiles/b14640b65a291b227cae93d513b0700e/recovered.edits/0000000000000000008 at 1731595948400 (+1 ms)Processed 5 edits across 1 Regions in 39 ms; skipped=3; WAL=hdfs://localhost:34445/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731595943987/wal.1731595944079, size=0, length=0, corrupted=false, cancelled=false at 1731595948400 2024-11-14T14:52:28,402 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:34445/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731595943987/wal.1731595944079 to hdfs://localhost:34445/hbase/oldWALs/wal.1731595944079 2024-11-14T14:52:28,403 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:34445/hbase/data/default/testCompactedBulkLoadedFiles/b14640b65a291b227cae93d513b0700e/recovered.edits/0000000000000000008 2024-11-14T14:52:28,403 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-14T14:52:28,405 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:34445/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731595943987, archiveDir=hdfs://localhost:34445/hbase/oldWALs, maxLogs=32 2024-11-14T14:52:28,418 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731595943987/wal.1731595948405, exclude list is [], retry=0 2024-11-14T14:52:28,420 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44549,DS-913adb4a-7946-4f46-896c-56bcdc4fcfa8,DISK] 2024-11-14T14:52:28,421 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42819,DS-c1b8ccdc-5e2c-4b3d-bb1e-6941b40dfb7e,DISK] 2024-11-14T14:52:28,421 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36189,DS-5c245456-9209-4937-bfad-43a9a1b09820,DISK] 2024-11-14T14:52:28,447 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731595943987/wal.1731595948405 2024-11-14T14:52:28,448 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33515:33515),(127.0.0.1/127.0.0.1:34777:34777),(127.0.0.1/127.0.0.1:33623:33623)] 2024-11-14T14:52:28,448 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => b14640b65a291b227cae93d513b0700e, NAME => 'testCompactedBulkLoadedFiles,,1731595943988.b14640b65a291b227cae93d513b0700e.', STARTKEY => '', ENDKEY => ''} 2024-11-14T14:52:28,448 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testCompactedBulkLoadedFiles,,1731595943988.b14640b65a291b227cae93d513b0700e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T14:52:28,448 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for b14640b65a291b227cae93d513b0700e 2024-11-14T14:52:28,448 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for b14640b65a291b227cae93d513b0700e 2024-11-14T14:52:28,450 INFO [StoreOpener-b14640b65a291b227cae93d513b0700e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region b14640b65a291b227cae93d513b0700e 2024-11-14T14:52:28,450 INFO [StoreOpener-b14640b65a291b227cae93d513b0700e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b14640b65a291b227cae93d513b0700e columnFamilyName a 2024-11-14T14:52:28,451 DEBUG [StoreOpener-b14640b65a291b227cae93d513b0700e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T14:52:28,457 DEBUG [StoreOpener-b14640b65a291b227cae93d513b0700e-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:34445/hbase/data/default/testCompactedBulkLoadedFiles/b14640b65a291b227cae93d513b0700e/a/12b9b2d9ff0e42c5a3d9981d6d08110c 2024-11-14T14:52:28,461 DEBUG [StoreFileOpener-b14640b65a291b227cae93d513b0700e-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for 3ded0b7ab55742a4b7c2ab80d1274a27_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-14T14:52:28,461 DEBUG [StoreOpener-b14640b65a291b227cae93d513b0700e-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:34445/hbase/data/default/testCompactedBulkLoadedFiles/b14640b65a291b227cae93d513b0700e/a/3ded0b7ab55742a4b7c2ab80d1274a27_SeqId_4_ 2024-11-14T14:52:28,465 DEBUG [StoreOpener-b14640b65a291b227cae93d513b0700e-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:34445/hbase/data/default/testCompactedBulkLoadedFiles/b14640b65a291b227cae93d513b0700e/a/8e021d58363449a7b4985dedd979cee3 2024-11-14T14:52:28,468 DEBUG [StoreFileOpener-b14640b65a291b227cae93d513b0700e-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for c9493fe89e7d4f4299249a6dfc961b6f_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-14T14:52:28,468 DEBUG [StoreOpener-b14640b65a291b227cae93d513b0700e-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:34445/hbase/data/default/testCompactedBulkLoadedFiles/b14640b65a291b227cae93d513b0700e/a/c9493fe89e7d4f4299249a6dfc961b6f_SeqId_4_ 2024-11-14T14:52:28,472 DEBUG [StoreFileOpener-b14640b65a291b227cae93d513b0700e-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for dc722b8c8301425996f7b1e6cbf6397c_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-14T14:52:28,472 DEBUG [StoreOpener-b14640b65a291b227cae93d513b0700e-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:34445/hbase/data/default/testCompactedBulkLoadedFiles/b14640b65a291b227cae93d513b0700e/a/dc722b8c8301425996f7b1e6cbf6397c_SeqId_4_ 2024-11-14T14:52:28,472 WARN [StoreOpener-b14640b65a291b227cae93d513b0700e-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:34445/hbase/data/default/testCompactedBulkLoadedFiles/b14640b65a291b227cae93d513b0700e/a/12b9b2d9ff0e42c5a3d9981d6d08110c from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@75a54b15 2024-11-14T14:52:28,472 WARN [StoreOpener-b14640b65a291b227cae93d513b0700e-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:34445/hbase/data/default/testCompactedBulkLoadedFiles/b14640b65a291b227cae93d513b0700e/a/3ded0b7ab55742a4b7c2ab80d1274a27_SeqId_4_ from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@75a54b15 2024-11-14T14:52:28,472 WARN [StoreOpener-b14640b65a291b227cae93d513b0700e-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:34445/hbase/data/default/testCompactedBulkLoadedFiles/b14640b65a291b227cae93d513b0700e/a/c9493fe89e7d4f4299249a6dfc961b6f_SeqId_4_ from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@75a54b15 2024-11-14T14:52:28,472 WARN [StoreOpener-b14640b65a291b227cae93d513b0700e-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:34445/hbase/data/default/testCompactedBulkLoadedFiles/b14640b65a291b227cae93d513b0700e/a/dc722b8c8301425996f7b1e6cbf6397c_SeqId_4_ from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@75a54b15 2024-11-14T14:52:28,472 DEBUG [StoreOpener-b14640b65a291b227cae93d513b0700e-1 {}] regionserver.StoreEngine(327): Moving the files [hdfs://localhost:34445/hbase/data/default/testCompactedBulkLoadedFiles/b14640b65a291b227cae93d513b0700e/a/12b9b2d9ff0e42c5a3d9981d6d08110c, hdfs://localhost:34445/hbase/data/default/testCompactedBulkLoadedFiles/b14640b65a291b227cae93d513b0700e/a/3ded0b7ab55742a4b7c2ab80d1274a27_SeqId_4_, hdfs://localhost:34445/hbase/data/default/testCompactedBulkLoadedFiles/b14640b65a291b227cae93d513b0700e/a/c9493fe89e7d4f4299249a6dfc961b6f_SeqId_4_, hdfs://localhost:34445/hbase/data/default/testCompactedBulkLoadedFiles/b14640b65a291b227cae93d513b0700e/a/dc722b8c8301425996f7b1e6cbf6397c_SeqId_4_] to archive 2024-11-14T14:52:28,473 DEBUG [StoreOpener-b14640b65a291b227cae93d513b0700e-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-14T14:52:28,475 DEBUG [StoreOpener-b14640b65a291b227cae93d513b0700e-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/hbase/data/default/testCompactedBulkLoadedFiles/b14640b65a291b227cae93d513b0700e/a/12b9b2d9ff0e42c5a3d9981d6d08110c to hdfs://localhost:34445/hbase/archive/data/default/testCompactedBulkLoadedFiles/b14640b65a291b227cae93d513b0700e/a/12b9b2d9ff0e42c5a3d9981d6d08110c 2024-11-14T14:52:28,476 DEBUG [StoreOpener-b14640b65a291b227cae93d513b0700e-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/hbase/data/default/testCompactedBulkLoadedFiles/b14640b65a291b227cae93d513b0700e/a/3ded0b7ab55742a4b7c2ab80d1274a27_SeqId_4_ to hdfs://localhost:34445/hbase/archive/data/default/testCompactedBulkLoadedFiles/b14640b65a291b227cae93d513b0700e/a/3ded0b7ab55742a4b7c2ab80d1274a27_SeqId_4_ 2024-11-14T14:52:28,477 DEBUG [StoreOpener-b14640b65a291b227cae93d513b0700e-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/hbase/data/default/testCompactedBulkLoadedFiles/b14640b65a291b227cae93d513b0700e/a/c9493fe89e7d4f4299249a6dfc961b6f_SeqId_4_ to hdfs://localhost:34445/hbase/archive/data/default/testCompactedBulkLoadedFiles/b14640b65a291b227cae93d513b0700e/a/c9493fe89e7d4f4299249a6dfc961b6f_SeqId_4_ 2024-11-14T14:52:28,478 DEBUG [StoreOpener-b14640b65a291b227cae93d513b0700e-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/hbase/data/default/testCompactedBulkLoadedFiles/b14640b65a291b227cae93d513b0700e/a/dc722b8c8301425996f7b1e6cbf6397c_SeqId_4_ to hdfs://localhost:34445/hbase/archive/data/default/testCompactedBulkLoadedFiles/b14640b65a291b227cae93d513b0700e/a/dc722b8c8301425996f7b1e6cbf6397c_SeqId_4_ 2024-11-14T14:52:28,478 INFO [StoreOpener-b14640b65a291b227cae93d513b0700e-1 {}] regionserver.HStore(327): Store=b14640b65a291b227cae93d513b0700e/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T14:52:28,478 INFO [StoreOpener-b14640b65a291b227cae93d513b0700e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region b14640b65a291b227cae93d513b0700e 2024-11-14T14:52:28,479 INFO [StoreOpener-b14640b65a291b227cae93d513b0700e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b14640b65a291b227cae93d513b0700e columnFamilyName b 2024-11-14T14:52:28,479 DEBUG [StoreOpener-b14640b65a291b227cae93d513b0700e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T14:52:28,479 INFO [StoreOpener-b14640b65a291b227cae93d513b0700e-1 {}] regionserver.HStore(327): Store=b14640b65a291b227cae93d513b0700e/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T14:52:28,480 INFO [StoreOpener-b14640b65a291b227cae93d513b0700e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region b14640b65a291b227cae93d513b0700e 2024-11-14T14:52:28,480 INFO [StoreOpener-b14640b65a291b227cae93d513b0700e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b14640b65a291b227cae93d513b0700e columnFamilyName c 2024-11-14T14:52:28,480 DEBUG [StoreOpener-b14640b65a291b227cae93d513b0700e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T14:52:28,481 INFO [StoreOpener-b14640b65a291b227cae93d513b0700e-1 {}] regionserver.HStore(327): Store=b14640b65a291b227cae93d513b0700e/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T14:52:28,481 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for b14640b65a291b227cae93d513b0700e 2024-11-14T14:52:28,482 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34445/hbase/data/default/testCompactedBulkLoadedFiles/b14640b65a291b227cae93d513b0700e 2024-11-14T14:52:28,483 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:34445/hbase/data/default/testCompactedBulkLoadedFiles/b14640b65a291b227cae93d513b0700e 2024-11-14T14:52:28,483 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:34445/hbase/data/default/testCompactedBulkLoadedFiles/b14640b65a291b227cae93d513b0700e/recovered.edits/0000000000000000008 2024-11-14T14:52:28,512 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-14T14:52:28,514 DEBUG [Time-limited test {}] regionserver.HRegion(5836): b14640b65a291b227cae93d513b0700e : Replaying compaction marker table_name: "testCompactedBulkLoadedFiles" encoded_region_name: "b14640b65a291b227cae93d513b0700e" family_name: "a" compaction_input: "12b9b2d9ff0e42c5a3d9981d6d08110c" compaction_input: "c9493fe89e7d4f4299249a6dfc961b6f_SeqId_4_" compaction_input: "3ded0b7ab55742a4b7c2ab80d1274a27_SeqId_4_" compaction_input: "dc722b8c8301425996f7b1e6cbf6397c_SeqId_4_" compaction_output: "8e021d58363449a7b4985dedd979cee3" store_home_dir: "a" region_name: "testCompactedBulkLoadedFiles,,1731595943988.b14640b65a291b227cae93d513b0700e." with seqId=9223372036854775807 and lastReplayedOpenRegionSeqId=-1 2024-11-14T14:52:28,514 DEBUG [Time-limited test {}] regionserver.HStore(1354): Completing compaction from the WAL marker 2024-11-14T14:52:28,514 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 0, skipped 2, firstSequenceIdInLog=3, maxSequenceIdInLog=8, path=hdfs://localhost:34445/hbase/data/default/testCompactedBulkLoadedFiles/b14640b65a291b227cae93d513b0700e/recovered.edits/0000000000000000008 2024-11-14T14:52:28,515 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:34445/hbase/data/default/testCompactedBulkLoadedFiles/b14640b65a291b227cae93d513b0700e/recovered.edits/0000000000000000008 2024-11-14T14:52:28,516 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for b14640b65a291b227cae93d513b0700e 2024-11-14T14:52:28,516 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for b14640b65a291b227cae93d513b0700e 2024-11-14T14:52:28,516 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testCompactedBulkLoadedFiles descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-14T14:52:28,518 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for b14640b65a291b227cae93d513b0700e 2024-11-14T14:52:28,520 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34445/hbase/data/default/testCompactedBulkLoadedFiles/b14640b65a291b227cae93d513b0700e/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-14T14:52:28,521 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened b14640b65a291b227cae93d513b0700e; next sequenceid=9; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70704354, jitterRate=0.05357697606086731}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-14T14:52:28,521 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for b14640b65a291b227cae93d513b0700e: Writing region info on filesystem at 1731595948448Initializing all the Stores at 1731595948449 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731595948449Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731595948449Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731595948449Cleaning up temporary data from old regions at 1731595948516 (+67 ms)Region opened successfully at 1731595948521 (+5 ms) 2024-11-14T14:52:28,524 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing b14640b65a291b227cae93d513b0700e, disabling compactions & flushes 2024-11-14T14:52:28,524 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testCompactedBulkLoadedFiles,,1731595943988.b14640b65a291b227cae93d513b0700e. 2024-11-14T14:52:28,524 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testCompactedBulkLoadedFiles,,1731595943988.b14640b65a291b227cae93d513b0700e. 2024-11-14T14:52:28,524 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testCompactedBulkLoadedFiles,,1731595943988.b14640b65a291b227cae93d513b0700e. after waiting 0 ms 2024-11-14T14:52:28,524 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testCompactedBulkLoadedFiles,,1731595943988.b14640b65a291b227cae93d513b0700e. 2024-11-14T14:52:28,525 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testCompactedBulkLoadedFiles,,1731595943988.b14640b65a291b227cae93d513b0700e. 2024-11-14T14:52:28,525 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for b14640b65a291b227cae93d513b0700e: Waiting for close lock at 1731595948524Disabling compacts and flushes for region at 1731595948524Disabling writes for close at 1731595948524Writing region close event to WAL at 1731595948525 (+1 ms)Closed at 1731595948525 2024-11-14T14:52:28,527 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42819 is added to blk_1073741924_1103 (size=175) 2024-11-14T14:52:28,527 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44549 is added to blk_1073741924_1103 (size=175) 2024-11-14T14:52:28,527 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36189 is added to blk_1073741924_1103 (size=175) 2024-11-14T14:52:28,530 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-14T14:52:28,530 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1731595948405) 2024-11-14T14:52:28,545 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testCompactedBulkLoadedFiles Thread=444 (was 429) Potentially hanging thread: AsyncFSWAL-26-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_314278571_22 at /127.0.0.1:43128 [Waiting for operation #9] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-26-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-26-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkinstestCompactedBulkLoadedFiles@localhost:34445 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1319272049) connection to localhost/127.0.0.1:34445 from jenkinstestCompactedBulkLoadedFiles java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_314278571_22 at /127.0.0.1:51870 [Waiting for operation #5] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_314278571_22 at /127.0.0.1:36946 [Waiting for operation #5] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1350 (was 1264) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=153 (was 158), ProcessCount=11 (was 11), AvailableMemoryMB=7521 (was 7522) 2024-11-14T14:52:28,545 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1350 is superior to 1024 2024-11-14T14:52:28,558 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsWrittenViaHRegion Thread=444, OpenFileDescriptor=1350, MaxFileDescriptor=1048576, SystemLoadAverage=153, ProcessCount=11, AvailableMemoryMB=7520 2024-11-14T14:52:28,559 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1350 is superior to 1024 2024-11-14T14:52:28,574 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-14T14:52:28,575 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-14T14:52:28,576 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-14T14:52:28,578 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-26932403, suffix=, logDir=hdfs://localhost:34445/hbase/WALs/hregion-26932403, archiveDir=hdfs://localhost:34445/hbase/oldWALs, maxLogs=32 2024-11-14T14:52:28,591 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-26932403/hregion-26932403.1731595948579, exclude list is [], retry=0 2024-11-14T14:52:28,594 DEBUG [AsyncFSWAL-28-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42819,DS-c1b8ccdc-5e2c-4b3d-bb1e-6941b40dfb7e,DISK] 2024-11-14T14:52:28,594 DEBUG [AsyncFSWAL-28-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36189,DS-5c245456-9209-4937-bfad-43a9a1b09820,DISK] 2024-11-14T14:52:28,594 DEBUG [AsyncFSWAL-28-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44549,DS-913adb4a-7946-4f46-896c-56bcdc4fcfa8,DISK] 2024-11-14T14:52:28,622 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-26932403/hregion-26932403.1731595948579 2024-11-14T14:52:28,622 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34777:34777),(127.0.0.1/127.0.0.1:33623:33623),(127.0.0.1/127.0.0.1:33515:33515)] 2024-11-14T14:52:28,622 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 585671c3116ed1daedec17be562d8e58, NAME => 'testReplayEditsWrittenViaHRegion,,1731595948574.585671c3116ed1daedec17be562d8e58.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenViaHRegion', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34445/hbase 2024-11-14T14:52:28,630 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36189 is added to blk_1073741926_1105 (size=67) 2024-11-14T14:52:28,630 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44549 is added to blk_1073741926_1105 (size=67) 2024-11-14T14:52:28,631 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42819 is added to blk_1073741926_1105 (size=67) 2024-11-14T14:52:28,631 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1731595948574.585671c3116ed1daedec17be562d8e58.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T14:52:28,632 INFO [StoreOpener-585671c3116ed1daedec17be562d8e58-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 585671c3116ed1daedec17be562d8e58 2024-11-14T14:52:28,634 INFO [StoreOpener-585671c3116ed1daedec17be562d8e58-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 585671c3116ed1daedec17be562d8e58 columnFamilyName a 2024-11-14T14:52:28,634 DEBUG [StoreOpener-585671c3116ed1daedec17be562d8e58-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T14:52:28,634 INFO [StoreOpener-585671c3116ed1daedec17be562d8e58-1 {}] regionserver.HStore(327): Store=585671c3116ed1daedec17be562d8e58/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T14:52:28,634 INFO [StoreOpener-585671c3116ed1daedec17be562d8e58-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 585671c3116ed1daedec17be562d8e58 2024-11-14T14:52:28,635 INFO [StoreOpener-585671c3116ed1daedec17be562d8e58-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 585671c3116ed1daedec17be562d8e58 columnFamilyName b 2024-11-14T14:52:28,636 DEBUG [StoreOpener-585671c3116ed1daedec17be562d8e58-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T14:52:28,636 INFO [StoreOpener-585671c3116ed1daedec17be562d8e58-1 {}] regionserver.HStore(327): Store=585671c3116ed1daedec17be562d8e58/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T14:52:28,636 INFO [StoreOpener-585671c3116ed1daedec17be562d8e58-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 585671c3116ed1daedec17be562d8e58 2024-11-14T14:52:28,637 INFO [StoreOpener-585671c3116ed1daedec17be562d8e58-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 585671c3116ed1daedec17be562d8e58 columnFamilyName c 2024-11-14T14:52:28,637 DEBUG [StoreOpener-585671c3116ed1daedec17be562d8e58-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T14:52:28,638 INFO [StoreOpener-585671c3116ed1daedec17be562d8e58-1 {}] regionserver.HStore(327): Store=585671c3116ed1daedec17be562d8e58/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T14:52:28,638 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 585671c3116ed1daedec17be562d8e58 2024-11-14T14:52:28,638 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34445/hbase/data/default/testReplayEditsWrittenViaHRegion/585671c3116ed1daedec17be562d8e58 2024-11-14T14:52:28,639 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34445/hbase/data/default/testReplayEditsWrittenViaHRegion/585671c3116ed1daedec17be562d8e58 2024-11-14T14:52:28,640 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 585671c3116ed1daedec17be562d8e58 2024-11-14T14:52:28,640 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 585671c3116ed1daedec17be562d8e58 2024-11-14T14:52:28,640 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-14T14:52:28,641 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 585671c3116ed1daedec17be562d8e58 2024-11-14T14:52:28,643 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34445/hbase/data/default/testReplayEditsWrittenViaHRegion/585671c3116ed1daedec17be562d8e58/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-14T14:52:28,644 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 585671c3116ed1daedec17be562d8e58; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62830865, jitterRate=-0.06374715268611908}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-14T14:52:28,644 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 585671c3116ed1daedec17be562d8e58: Writing region info on filesystem at 1731595948631Initializing all the Stores at 1731595948632 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731595948632Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731595948632Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731595948632Cleaning up temporary data from old regions at 1731595948640 (+8 ms)Region opened successfully at 1731595948644 (+4 ms) 2024-11-14T14:52:28,644 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 585671c3116ed1daedec17be562d8e58, disabling compactions & flushes 2024-11-14T14:52:28,644 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1731595948574.585671c3116ed1daedec17be562d8e58. 2024-11-14T14:52:28,644 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1731595948574.585671c3116ed1daedec17be562d8e58. 2024-11-14T14:52:28,644 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1731595948574.585671c3116ed1daedec17be562d8e58. after waiting 0 ms 2024-11-14T14:52:28,644 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1731595948574.585671c3116ed1daedec17be562d8e58. 2024-11-14T14:52:28,645 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1731595948574.585671c3116ed1daedec17be562d8e58. 2024-11-14T14:52:28,645 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 585671c3116ed1daedec17be562d8e58: Waiting for close lock at 1731595948644Disabling compacts and flushes for region at 1731595948644Disabling writes for close at 1731595948644Writing region close event to WAL at 1731595948644Closed at 1731595948645 (+1 ms) 2024-11-14T14:52:28,647 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44549 is added to blk_1073741925_1104 (size=175) 2024-11-14T14:52:28,647 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36189 is added to blk_1073741925_1104 (size=175) 2024-11-14T14:52:28,647 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42819 is added to blk_1073741925_1104 (size=175) 2024-11-14T14:52:28,649 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-14T14:52:28,649 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-26932403:(num 1731595948579) 2024-11-14T14:52:28,650 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-14T14:52:28,652 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:34445/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731595948573, archiveDir=hdfs://localhost:34445/hbase/oldWALs, maxLogs=32 2024-11-14T14:52:28,665 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731595948573/wal.1731595948652, exclude list is [], retry=0 2024-11-14T14:52:28,668 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44549,DS-913adb4a-7946-4f46-896c-56bcdc4fcfa8,DISK] 2024-11-14T14:52:28,668 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36189,DS-5c245456-9209-4937-bfad-43a9a1b09820,DISK] 2024-11-14T14:52:28,669 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42819,DS-c1b8ccdc-5e2c-4b3d-bb1e-6941b40dfb7e,DISK] 2024-11-14T14:52:28,696 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731595948573/wal.1731595948652 2024-11-14T14:52:28,696 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33515:33515),(127.0.0.1/127.0.0.1:33623:33623),(127.0.0.1/127.0.0.1:34777:34777)] 2024-11-14T14:52:28,697 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 585671c3116ed1daedec17be562d8e58, NAME => 'testReplayEditsWrittenViaHRegion,,1731595948574.585671c3116ed1daedec17be562d8e58.', STARTKEY => '', ENDKEY => ''} 2024-11-14T14:52:28,697 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1731595948574.585671c3116ed1daedec17be562d8e58.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T14:52:28,697 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 585671c3116ed1daedec17be562d8e58 2024-11-14T14:52:28,697 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 585671c3116ed1daedec17be562d8e58 2024-11-14T14:52:28,699 INFO [StoreOpener-585671c3116ed1daedec17be562d8e58-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 585671c3116ed1daedec17be562d8e58 2024-11-14T14:52:28,700 INFO [StoreOpener-585671c3116ed1daedec17be562d8e58-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 585671c3116ed1daedec17be562d8e58 columnFamilyName a 2024-11-14T14:52:28,700 DEBUG [StoreOpener-585671c3116ed1daedec17be562d8e58-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T14:52:28,700 INFO [StoreOpener-585671c3116ed1daedec17be562d8e58-1 {}] regionserver.HStore(327): Store=585671c3116ed1daedec17be562d8e58/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T14:52:28,701 INFO [StoreOpener-585671c3116ed1daedec17be562d8e58-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 585671c3116ed1daedec17be562d8e58 2024-11-14T14:52:28,701 INFO [StoreOpener-585671c3116ed1daedec17be562d8e58-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 585671c3116ed1daedec17be562d8e58 columnFamilyName b 2024-11-14T14:52:28,702 DEBUG [StoreOpener-585671c3116ed1daedec17be562d8e58-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T14:52:28,702 INFO [StoreOpener-585671c3116ed1daedec17be562d8e58-1 {}] regionserver.HStore(327): Store=585671c3116ed1daedec17be562d8e58/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T14:52:28,702 INFO [StoreOpener-585671c3116ed1daedec17be562d8e58-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 585671c3116ed1daedec17be562d8e58 2024-11-14T14:52:28,703 INFO [StoreOpener-585671c3116ed1daedec17be562d8e58-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 585671c3116ed1daedec17be562d8e58 columnFamilyName c 2024-11-14T14:52:28,703 DEBUG [StoreOpener-585671c3116ed1daedec17be562d8e58-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T14:52:28,703 INFO [StoreOpener-585671c3116ed1daedec17be562d8e58-1 {}] regionserver.HStore(327): Store=585671c3116ed1daedec17be562d8e58/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T14:52:28,703 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 585671c3116ed1daedec17be562d8e58 2024-11-14T14:52:28,704 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34445/hbase/data/default/testReplayEditsWrittenViaHRegion/585671c3116ed1daedec17be562d8e58 2024-11-14T14:52:28,705 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34445/hbase/data/default/testReplayEditsWrittenViaHRegion/585671c3116ed1daedec17be562d8e58 2024-11-14T14:52:28,706 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 585671c3116ed1daedec17be562d8e58 2024-11-14T14:52:28,706 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 585671c3116ed1daedec17be562d8e58 2024-11-14T14:52:28,707 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-14T14:52:28,708 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 585671c3116ed1daedec17be562d8e58 2024-11-14T14:52:28,709 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 585671c3116ed1daedec17be562d8e58; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71869295, jitterRate=0.07093594968318939}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-14T14:52:28,709 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 585671c3116ed1daedec17be562d8e58: Writing region info on filesystem at 1731595948697Initializing all the Stores at 1731595948698 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731595948698Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731595948698Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731595948698Cleaning up temporary data from old regions at 1731595948706 (+8 ms)Region opened successfully at 1731595948709 (+3 ms) 2024-11-14T14:52:28,717 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 585671c3116ed1daedec17be562d8e58 3/3 column families, dataSize=870 B heapSize=2.31 KB 2024-11-14T14:52:28,733 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/hbase/data/default/testReplayEditsWrittenViaHRegion/585671c3116ed1daedec17be562d8e58/.tmp/a/96be49394a3a450a8db8918e9267c9b2 is 91, key is testReplayEditsWrittenViaHRegion/a:x0/1731595948709/Put/seqid=0 2024-11-14T14:52:28,740 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36189 is added to blk_1073741928_1107 (size=5958) 2024-11-14T14:52:28,740 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42819 is added to blk_1073741928_1107 (size=5958) 2024-11-14T14:52:28,740 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44549 is added to blk_1073741928_1107 (size=5958) 2024-11-14T14:52:28,741 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:34445/hbase/data/default/testReplayEditsWrittenViaHRegion/585671c3116ed1daedec17be562d8e58/.tmp/a/96be49394a3a450a8db8918e9267c9b2 2024-11-14T14:52:28,747 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/hbase/data/default/testReplayEditsWrittenViaHRegion/585671c3116ed1daedec17be562d8e58/.tmp/a/96be49394a3a450a8db8918e9267c9b2 as hdfs://localhost:34445/hbase/data/default/testReplayEditsWrittenViaHRegion/585671c3116ed1daedec17be562d8e58/a/96be49394a3a450a8db8918e9267c9b2 2024-11-14T14:52:28,753 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/hbase/data/default/testReplayEditsWrittenViaHRegion/585671c3116ed1daedec17be562d8e58/a/96be49394a3a450a8db8918e9267c9b2, entries=10, sequenceid=13, filesize=5.8 K 2024-11-14T14:52:28,755 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~870 B/870, heapSize ~1.80 KB/1840, currentSize=0 B/0 for 585671c3116ed1daedec17be562d8e58 in 38ms, sequenceid=13, compaction requested=false 2024-11-14T14:52:28,755 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 585671c3116ed1daedec17be562d8e58: 2024-11-14T14:52:28,775 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 585671c3116ed1daedec17be562d8e58, disabling compactions & flushes 2024-11-14T14:52:28,776 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1731595948574.585671c3116ed1daedec17be562d8e58. 2024-11-14T14:52:28,776 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1731595948574.585671c3116ed1daedec17be562d8e58. 2024-11-14T14:52:28,776 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1731595948574.585671c3116ed1daedec17be562d8e58. after waiting 0 ms 2024-11-14T14:52:28,776 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1731595948574.585671c3116ed1daedec17be562d8e58. 2024-11-14T14:52:28,777 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 1740 in region testReplayEditsWrittenViaHRegion,,1731595948574.585671c3116ed1daedec17be562d8e58. 2024-11-14T14:52:28,777 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1731595948574.585671c3116ed1daedec17be562d8e58. 2024-11-14T14:52:28,777 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 585671c3116ed1daedec17be562d8e58: Waiting for close lock at 1731595948775Disabling compacts and flushes for region at 1731595948775Disabling writes for close at 1731595948776 (+1 ms)Writing region close event to WAL at 1731595948777 (+1 ms)Closed at 1731595948777 2024-11-14T14:52:28,779 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42819 is added to blk_1073741927_1106 (size=6243) 2024-11-14T14:52:28,779 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36189 is added to blk_1073741927_1106 (size=6243) 2024-11-14T14:52:28,779 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44549 is added to blk_1073741927_1106 (size=6243) 2024-11-14T14:52:28,797 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:34445/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731595948573/wal.1731595948652, size=6.1 K (6243bytes) 2024-11-14T14:52:28,797 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:34445/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731595948573/wal.1731595948652 2024-11-14T14:52:28,798 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:34445/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731595948573/wal.1731595948652 after 1ms 2024-11-14T14:52:28,826 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-14T14:52:28,826 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:34445/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731595948573/wal.1731595948652 took 29ms 2024-11-14T14:52:28,829 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:34445/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731595948573/wal.1731595948652 so closing down 2024-11-14T14:52:28,829 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-14T14:52:28,830 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1731595948652.temp 2024-11-14T14:52:28,858 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:34445/hbase/data/default/testReplayEditsWrittenViaHRegion/585671c3116ed1daedec17be562d8e58/recovered.edits/0000000000000000003-wal.1731595948652.temp 2024-11-14T14:52:28,859 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-14T14:52:28,865 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44549 is added to blk_1073741929_1108 (size=5575) 2024-11-14T14:52:28,865 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36189 is added to blk_1073741929_1108 (size=5575) 2024-11-14T14:52:28,866 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42819 is added to blk_1073741929_1108 (size=5575) 2024-11-14T14:52:28,867 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:34445/hbase/data/default/testReplayEditsWrittenViaHRegion/585671c3116ed1daedec17be562d8e58/recovered.edits/0000000000000000003-wal.1731595948652.temp (wrote 30 edits, skipped 0 edits in 0 ms) 2024-11-14T14:52:28,868 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:34445/hbase/data/default/testReplayEditsWrittenViaHRegion/585671c3116ed1daedec17be562d8e58/recovered.edits/0000000000000000003-wal.1731595948652.temp to hdfs://localhost:34445/hbase/data/default/testReplayEditsWrittenViaHRegion/585671c3116ed1daedec17be562d8e58/recovered.edits/0000000000000000035 2024-11-14T14:52:28,868 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 32 edits across 1 Regions in 41 ms; skipped=2; WAL=hdfs://localhost:34445/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731595948573/wal.1731595948652, size=6.1 K, length=6243, corrupted=false, cancelled=false 2024-11-14T14:52:28,868 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:34445/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731595948573/wal.1731595948652, journal: Splitting hdfs://localhost:34445/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731595948573/wal.1731595948652, size=6.1 K (6243bytes) at 1731595948797Finishing writing output for hdfs://localhost:34445/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731595948573/wal.1731595948652 so closing down at 1731595948829 (+32 ms)Creating recovered edits writer path=hdfs://localhost:34445/hbase/data/default/testReplayEditsWrittenViaHRegion/585671c3116ed1daedec17be562d8e58/recovered.edits/0000000000000000003-wal.1731595948652.temp at 1731595948858 (+29 ms)3 split writer threads finished at 1731595948859 (+1 ms)Closed recovered edits writer path=hdfs://localhost:34445/hbase/data/default/testReplayEditsWrittenViaHRegion/585671c3116ed1daedec17be562d8e58/recovered.edits/0000000000000000003-wal.1731595948652.temp (wrote 30 edits, skipped 0 edits in 0 ms) at 1731595948867 (+8 ms)Rename recovered edits hdfs://localhost:34445/hbase/data/default/testReplayEditsWrittenViaHRegion/585671c3116ed1daedec17be562d8e58/recovered.edits/0000000000000000003-wal.1731595948652.temp to hdfs://localhost:34445/hbase/data/default/testReplayEditsWrittenViaHRegion/585671c3116ed1daedec17be562d8e58/recovered.edits/0000000000000000035 at 1731595948868 (+1 ms)Processed 32 edits across 1 Regions in 41 ms; skipped=2; WAL=hdfs://localhost:34445/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731595948573/wal.1731595948652, size=6.1 K, length=6243, corrupted=false, cancelled=false at 1731595948868 2024-11-14T14:52:28,870 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:34445/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731595948573/wal.1731595948652 to hdfs://localhost:34445/hbase/oldWALs/wal.1731595948652 2024-11-14T14:52:28,871 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:34445/hbase/data/default/testReplayEditsWrittenViaHRegion/585671c3116ed1daedec17be562d8e58/recovered.edits/0000000000000000035 2024-11-14T14:52:28,871 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-14T14:52:28,873 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:34445/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731595948573, archiveDir=hdfs://localhost:34445/hbase/oldWALs, maxLogs=32 2024-11-14T14:52:28,887 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731595948573/wal.1731595948873, exclude list is [], retry=0 2024-11-14T14:52:28,889 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42819,DS-c1b8ccdc-5e2c-4b3d-bb1e-6941b40dfb7e,DISK] 2024-11-14T14:52:28,890 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36189,DS-5c245456-9209-4937-bfad-43a9a1b09820,DISK] 2024-11-14T14:52:28,890 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44549,DS-913adb4a-7946-4f46-896c-56bcdc4fcfa8,DISK] 2024-11-14T14:52:28,918 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731595948573/wal.1731595948873 2024-11-14T14:52:28,918 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34777:34777),(127.0.0.1/127.0.0.1:33623:33623),(127.0.0.1/127.0.0.1:33515:33515)] 2024-11-14T14:52:28,918 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 585671c3116ed1daedec17be562d8e58, NAME => 'testReplayEditsWrittenViaHRegion,,1731595948574.585671c3116ed1daedec17be562d8e58.', STARTKEY => '', ENDKEY => ''} 2024-11-14T14:52:28,918 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1731595948574.585671c3116ed1daedec17be562d8e58.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T14:52:28,918 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 585671c3116ed1daedec17be562d8e58 2024-11-14T14:52:28,918 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 585671c3116ed1daedec17be562d8e58 2024-11-14T14:52:28,920 INFO [StoreOpener-585671c3116ed1daedec17be562d8e58-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 585671c3116ed1daedec17be562d8e58 2024-11-14T14:52:28,921 INFO [StoreOpener-585671c3116ed1daedec17be562d8e58-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 585671c3116ed1daedec17be562d8e58 columnFamilyName a 2024-11-14T14:52:28,921 DEBUG [StoreOpener-585671c3116ed1daedec17be562d8e58-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T14:52:28,926 DEBUG [StoreOpener-585671c3116ed1daedec17be562d8e58-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:34445/hbase/data/default/testReplayEditsWrittenViaHRegion/585671c3116ed1daedec17be562d8e58/a/96be49394a3a450a8db8918e9267c9b2 2024-11-14T14:52:28,926 INFO [StoreOpener-585671c3116ed1daedec17be562d8e58-1 {}] regionserver.HStore(327): Store=585671c3116ed1daedec17be562d8e58/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T14:52:28,927 INFO [StoreOpener-585671c3116ed1daedec17be562d8e58-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 585671c3116ed1daedec17be562d8e58 2024-11-14T14:52:28,927 INFO [StoreOpener-585671c3116ed1daedec17be562d8e58-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 585671c3116ed1daedec17be562d8e58 columnFamilyName b 2024-11-14T14:52:28,927 DEBUG [StoreOpener-585671c3116ed1daedec17be562d8e58-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T14:52:28,928 INFO [StoreOpener-585671c3116ed1daedec17be562d8e58-1 {}] regionserver.HStore(327): Store=585671c3116ed1daedec17be562d8e58/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T14:52:28,928 INFO [StoreOpener-585671c3116ed1daedec17be562d8e58-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 585671c3116ed1daedec17be562d8e58 2024-11-14T14:52:28,928 INFO [StoreOpener-585671c3116ed1daedec17be562d8e58-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 585671c3116ed1daedec17be562d8e58 columnFamilyName c 2024-11-14T14:52:28,928 DEBUG [StoreOpener-585671c3116ed1daedec17be562d8e58-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T14:52:28,929 INFO [StoreOpener-585671c3116ed1daedec17be562d8e58-1 {}] regionserver.HStore(327): Store=585671c3116ed1daedec17be562d8e58/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T14:52:28,929 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 585671c3116ed1daedec17be562d8e58 2024-11-14T14:52:28,929 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34445/hbase/data/default/testReplayEditsWrittenViaHRegion/585671c3116ed1daedec17be562d8e58 2024-11-14T14:52:28,931 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:34445/hbase/data/default/testReplayEditsWrittenViaHRegion/585671c3116ed1daedec17be562d8e58 2024-11-14T14:52:28,931 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:34445/hbase/data/default/testReplayEditsWrittenViaHRegion/585671c3116ed1daedec17be562d8e58/recovered.edits/0000000000000000035 2024-11-14T14:52:28,958 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-14T14:52:28,959 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 20, skipped 10, firstSequenceIdInLog=3, maxSequenceIdInLog=35, path=hdfs://localhost:34445/hbase/data/default/testReplayEditsWrittenViaHRegion/585671c3116ed1daedec17be562d8e58/recovered.edits/0000000000000000035 2024-11-14T14:52:28,959 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 585671c3116ed1daedec17be562d8e58 3/3 column families, dataSize=1.70 KB heapSize=3.88 KB 2024-11-14T14:52:28,973 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/hbase/data/default/testReplayEditsWrittenViaHRegion/585671c3116ed1daedec17be562d8e58/.tmp/b/44638afb11124c4e94ce7658c434f38e is 91, key is testReplayEditsWrittenViaHRegion/b:x0/1731595948755/Put/seqid=0 2024-11-14T14:52:28,979 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42819 is added to blk_1073741931_1110 (size=5958) 2024-11-14T14:52:28,979 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44549 is added to blk_1073741931_1110 (size=5958) 2024-11-14T14:52:28,980 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36189 is added to blk_1073741931_1110 (size=5958) 2024-11-14T14:52:28,980 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=35 (bloomFilter=true), to=hdfs://localhost:34445/hbase/data/default/testReplayEditsWrittenViaHRegion/585671c3116ed1daedec17be562d8e58/.tmp/b/44638afb11124c4e94ce7658c434f38e 2024-11-14T14:52:28,998 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/hbase/data/default/testReplayEditsWrittenViaHRegion/585671c3116ed1daedec17be562d8e58/.tmp/c/c21f9828d72b47adbc6b127bd274a7e5 is 91, key is testReplayEditsWrittenViaHRegion/c:x0/1731595948764/Put/seqid=0 2024-11-14T14:52:29,004 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42819 is added to blk_1073741932_1111 (size=5958) 2024-11-14T14:52:29,005 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36189 is added to blk_1073741932_1111 (size=5958) 2024-11-14T14:52:29,005 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44549 is added to blk_1073741932_1111 (size=5958) 2024-11-14T14:52:29,005 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=35 (bloomFilter=true), to=hdfs://localhost:34445/hbase/data/default/testReplayEditsWrittenViaHRegion/585671c3116ed1daedec17be562d8e58/.tmp/c/c21f9828d72b47adbc6b127bd274a7e5 2024-11-14T14:52:29,011 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/hbase/data/default/testReplayEditsWrittenViaHRegion/585671c3116ed1daedec17be562d8e58/.tmp/b/44638afb11124c4e94ce7658c434f38e as hdfs://localhost:34445/hbase/data/default/testReplayEditsWrittenViaHRegion/585671c3116ed1daedec17be562d8e58/b/44638afb11124c4e94ce7658c434f38e 2024-11-14T14:52:29,017 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/hbase/data/default/testReplayEditsWrittenViaHRegion/585671c3116ed1daedec17be562d8e58/b/44638afb11124c4e94ce7658c434f38e, entries=10, sequenceid=35, filesize=5.8 K 2024-11-14T14:52:29,018 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/hbase/data/default/testReplayEditsWrittenViaHRegion/585671c3116ed1daedec17be562d8e58/.tmp/c/c21f9828d72b47adbc6b127bd274a7e5 as hdfs://localhost:34445/hbase/data/default/testReplayEditsWrittenViaHRegion/585671c3116ed1daedec17be562d8e58/c/c21f9828d72b47adbc6b127bd274a7e5 2024-11-14T14:52:29,022 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/hbase/data/default/testReplayEditsWrittenViaHRegion/585671c3116ed1daedec17be562d8e58/c/c21f9828d72b47adbc6b127bd274a7e5, entries=10, sequenceid=35, filesize=5.8 K 2024-11-14T14:52:29,022 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~1.70 KB/1740, heapSize ~3.59 KB/3680, currentSize=0 B/0 for 585671c3116ed1daedec17be562d8e58 in 63ms, sequenceid=35, compaction requested=false; wal=null 2024-11-14T14:52:29,023 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:34445/hbase/data/default/testReplayEditsWrittenViaHRegion/585671c3116ed1daedec17be562d8e58/recovered.edits/0000000000000000035 2024-11-14T14:52:29,024 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 585671c3116ed1daedec17be562d8e58 2024-11-14T14:52:29,024 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 585671c3116ed1daedec17be562d8e58 2024-11-14T14:52:29,025 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-14T14:52:29,026 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 585671c3116ed1daedec17be562d8e58 2024-11-14T14:52:29,028 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34445/hbase/data/default/testReplayEditsWrittenViaHRegion/585671c3116ed1daedec17be562d8e58/recovered.edits/35.seqid, newMaxSeqId=35, maxSeqId=1 2024-11-14T14:52:29,029 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 585671c3116ed1daedec17be562d8e58; next sequenceid=36; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75174731, jitterRate=0.12019078433513641}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-14T14:52:29,029 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 585671c3116ed1daedec17be562d8e58: Writing region info on filesystem at 1731595948918Initializing all the Stores at 1731595948919 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731595948919Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731595948920 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731595948920Obtaining lock to block concurrent updates at 1731595948959 (+39 ms)Preparing flush snapshotting stores in 585671c3116ed1daedec17be562d8e58 at 1731595948959Finished memstore snapshotting testReplayEditsWrittenViaHRegion,,1731595948574.585671c3116ed1daedec17be562d8e58., syncing WAL and waiting on mvcc, flushsize=dataSize=1740, getHeapSize=3920, getOffHeapSize=0, getCellsCount=20 at 1731595948959Flushing stores of testReplayEditsWrittenViaHRegion,,1731595948574.585671c3116ed1daedec17be562d8e58. at 1731595948959Flushing 585671c3116ed1daedec17be562d8e58/b: creating writer at 1731595948959Flushing 585671c3116ed1daedec17be562d8e58/b: appending metadata at 1731595948972 (+13 ms)Flushing 585671c3116ed1daedec17be562d8e58/b: closing flushed file at 1731595948972Flushing 585671c3116ed1daedec17be562d8e58/c: creating writer at 1731595948985 (+13 ms)Flushing 585671c3116ed1daedec17be562d8e58/c: appending metadata at 1731595948998 (+13 ms)Flushing 585671c3116ed1daedec17be562d8e58/c: closing flushed file at 1731595948998Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@197364cd: reopening flushed file at 1731595949010 (+12 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@797a22b6: reopening flushed file at 1731595949017 (+7 ms)Finished flush of dataSize ~1.70 KB/1740, heapSize ~3.59 KB/3680, currentSize=0 B/0 for 585671c3116ed1daedec17be562d8e58 in 63ms, sequenceid=35, compaction requested=false; wal=null at 1731595949022 (+5 ms)Cleaning up temporary data from old regions at 1731595949024 (+2 ms)Region opened successfully at 1731595949029 (+5 ms) 2024-11-14T14:52:29,095 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:34445/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731595948573/wal.1731595948873, size=0 (0bytes) 2024-11-14T14:52:29,095 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:34445/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731595948573/wal.1731595948873 might be still open, length is 0 2024-11-14T14:52:29,095 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:34445/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731595948573/wal.1731595948873 2024-11-14T14:52:29,095 WARN [IPC Server handler 2 on default port 34445 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731595948573/wal.1731595948873 has not been closed. Lease recovery is in progress. RecoveryId = 1112 for block blk_1073741930_1109 2024-11-14T14:52:29,096 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:34445/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731595948573/wal.1731595948873 after 1ms 2024-11-14T14:52:30,689 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1791616411_22 at /127.0.0.1:37012 [Receiving block BP-1055376130-172.17.0.2-1731595909704:blk_1073741930_1109] {}] datanode.DataXceiver(331): 127.0.0.1:42819:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:37012 dst: /127.0.0.1:42819 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:42819 remote=/127.0.0.1:37012]. Total timeout mills is 60000, 58366 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:52:30,690 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1791616411_22 at /127.0.0.1:43186 [Receiving block BP-1055376130-172.17.0.2-1731595909704:blk_1073741930_1109] {}] datanode.DataXceiver(331): 127.0.0.1:44549:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:43186 dst: /127.0.0.1:44549 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:52:30,690 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1791616411_22 at /127.0.0.1:51918 [Receiving block BP-1055376130-172.17.0.2-1731595909704:blk_1073741930_1109] {}] datanode.DataXceiver(331): 127.0.0.1:36189:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:51918 dst: /127.0.0.1:36189 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:52:30,693 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42819 is added to blk_1073741930_1112 (size=5567) 2024-11-14T14:52:30,693 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36189 is added to blk_1073741930_1112 (size=5567) 2024-11-14T14:52:30,694 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44549 is added to blk_1073741930_1112 (size=5567) 2024-11-14T14:52:33,096 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:34445/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731595948573/wal.1731595948873 after 4001ms 2024-11-14T14:52:33,124 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-14T14:52:33,124 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:34445/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731595948573/wal.1731595948873 took 4030ms 2024-11-14T14:52:33,126 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:34445/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731595948573/wal.1731595948873; continuing. 2024-11-14T14:52:33,126 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:34445/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731595948573/wal.1731595948873 so closing down 2024-11-14T14:52:33,126 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-14T14:52:33,127 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000037-wal.1731595948873.temp 2024-11-14T14:52:33,152 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testDatalossWhenInputError 2024-11-14T14:52:33,152 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testDatalossWhenInputError Metrics about Tables on a single HBase RegionServer 2024-11-14T14:52:33,153 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:34445/hbase/data/default/testReplayEditsWrittenViaHRegion/585671c3116ed1daedec17be562d8e58/recovered.edits/0000000000000000037-wal.1731595948873.temp 2024-11-14T14:52:33,154 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-14T14:52:33,154 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testCompactedBulkLoadedFiles 2024-11-14T14:52:33,154 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testCompactedBulkLoadedFiles Metrics about Tables on a single HBase RegionServer 2024-11-14T14:52:33,159 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42819 is added to blk_1073741933_1113 (size=5575) 2024-11-14T14:52:33,160 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36189 is added to blk_1073741933_1113 (size=5575) 2024-11-14T14:52:33,160 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44549 is added to blk_1073741933_1113 (size=5575) 2024-11-14T14:52:33,163 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:34445/hbase/data/default/testReplayEditsWrittenViaHRegion/585671c3116ed1daedec17be562d8e58/recovered.edits/0000000000000000037-wal.1731595948873.temp (wrote 30 edits, skipped 0 edits in 0 ms) 2024-11-14T14:52:33,164 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:34445/hbase/data/default/testReplayEditsWrittenViaHRegion/585671c3116ed1daedec17be562d8e58/recovered.edits/0000000000000000037-wal.1731595948873.temp to hdfs://localhost:34445/hbase/data/default/testReplayEditsWrittenViaHRegion/585671c3116ed1daedec17be562d8e58/recovered.edits/0000000000000000066 2024-11-14T14:52:33,164 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 30 edits across 1 Regions in 40 ms; skipped=0; WAL=hdfs://localhost:34445/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731595948573/wal.1731595948873, size=0, length=0, corrupted=false, cancelled=false 2024-11-14T14:52:33,164 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:34445/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731595948573/wal.1731595948873, journal: Splitting hdfs://localhost:34445/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731595948573/wal.1731595948873, size=0 (0bytes) at 1731595949095Finishing writing output for hdfs://localhost:34445/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731595948573/wal.1731595948873 so closing down at 1731595953126 (+4031 ms)Creating recovered edits writer path=hdfs://localhost:34445/hbase/data/default/testReplayEditsWrittenViaHRegion/585671c3116ed1daedec17be562d8e58/recovered.edits/0000000000000000037-wal.1731595948873.temp at 1731595953153 (+27 ms)3 split writer threads finished at 1731595953154 (+1 ms)Closed recovered edits writer path=hdfs://localhost:34445/hbase/data/default/testReplayEditsWrittenViaHRegion/585671c3116ed1daedec17be562d8e58/recovered.edits/0000000000000000037-wal.1731595948873.temp (wrote 30 edits, skipped 0 edits in 0 ms) at 1731595953163 (+9 ms)Rename recovered edits hdfs://localhost:34445/hbase/data/default/testReplayEditsWrittenViaHRegion/585671c3116ed1daedec17be562d8e58/recovered.edits/0000000000000000037-wal.1731595948873.temp to hdfs://localhost:34445/hbase/data/default/testReplayEditsWrittenViaHRegion/585671c3116ed1daedec17be562d8e58/recovered.edits/0000000000000000066 at 1731595953164 (+1 ms)Processed 30 edits across 1 Regions in 40 ms; skipped=0; WAL=hdfs://localhost:34445/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731595948573/wal.1731595948873, size=0, length=0, corrupted=false, cancelled=false at 1731595953164 2024-11-14T14:52:33,166 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:34445/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731595948573/wal.1731595948873 to hdfs://localhost:34445/hbase/oldWALs/wal.1731595948873 2024-11-14T14:52:33,166 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:34445/hbase/data/default/testReplayEditsWrittenViaHRegion/585671c3116ed1daedec17be562d8e58/recovered.edits/0000000000000000066 2024-11-14T14:52:33,167 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-14T14:52:33,168 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:34445/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731595948573, archiveDir=hdfs://localhost:34445/hbase/oldWALs, maxLogs=32 2024-11-14T14:52:33,181 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731595948573/wal.1731595953168, exclude list is [], retry=0 2024-11-14T14:52:33,183 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36189,DS-5c245456-9209-4937-bfad-43a9a1b09820,DISK] 2024-11-14T14:52:33,183 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44549,DS-913adb4a-7946-4f46-896c-56bcdc4fcfa8,DISK] 2024-11-14T14:52:33,183 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42819,DS-c1b8ccdc-5e2c-4b3d-bb1e-6941b40dfb7e,DISK] 2024-11-14T14:52:33,210 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731595948573/wal.1731595953168 2024-11-14T14:52:33,210 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33623:33623),(127.0.0.1/127.0.0.1:33515:33515),(127.0.0.1/127.0.0.1:34777:34777)] 2024-11-14T14:52:33,210 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1731595948574.585671c3116ed1daedec17be562d8e58.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T14:52:33,211 INFO [StoreOpener-585671c3116ed1daedec17be562d8e58-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 585671c3116ed1daedec17be562d8e58 2024-11-14T14:52:33,212 INFO [StoreOpener-585671c3116ed1daedec17be562d8e58-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 585671c3116ed1daedec17be562d8e58 columnFamilyName a 2024-11-14T14:52:33,212 DEBUG [StoreOpener-585671c3116ed1daedec17be562d8e58-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T14:52:33,217 DEBUG [StoreOpener-585671c3116ed1daedec17be562d8e58-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:34445/hbase/data/default/testReplayEditsWrittenViaHRegion/585671c3116ed1daedec17be562d8e58/a/96be49394a3a450a8db8918e9267c9b2 2024-11-14T14:52:33,217 INFO [StoreOpener-585671c3116ed1daedec17be562d8e58-1 {}] regionserver.HStore(327): Store=585671c3116ed1daedec17be562d8e58/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T14:52:33,217 INFO [StoreOpener-585671c3116ed1daedec17be562d8e58-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 585671c3116ed1daedec17be562d8e58 2024-11-14T14:52:33,218 INFO [StoreOpener-585671c3116ed1daedec17be562d8e58-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 585671c3116ed1daedec17be562d8e58 columnFamilyName b 2024-11-14T14:52:33,218 DEBUG [StoreOpener-585671c3116ed1daedec17be562d8e58-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T14:52:33,222 DEBUG [StoreOpener-585671c3116ed1daedec17be562d8e58-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:34445/hbase/data/default/testReplayEditsWrittenViaHRegion/585671c3116ed1daedec17be562d8e58/b/44638afb11124c4e94ce7658c434f38e 2024-11-14T14:52:33,222 INFO [StoreOpener-585671c3116ed1daedec17be562d8e58-1 {}] regionserver.HStore(327): Store=585671c3116ed1daedec17be562d8e58/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T14:52:33,222 INFO [StoreOpener-585671c3116ed1daedec17be562d8e58-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 585671c3116ed1daedec17be562d8e58 2024-11-14T14:52:33,223 INFO [StoreOpener-585671c3116ed1daedec17be562d8e58-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 585671c3116ed1daedec17be562d8e58 columnFamilyName c 2024-11-14T14:52:33,223 DEBUG [StoreOpener-585671c3116ed1daedec17be562d8e58-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T14:52:33,227 DEBUG [StoreOpener-585671c3116ed1daedec17be562d8e58-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:34445/hbase/data/default/testReplayEditsWrittenViaHRegion/585671c3116ed1daedec17be562d8e58/c/c21f9828d72b47adbc6b127bd274a7e5 2024-11-14T14:52:33,227 INFO [StoreOpener-585671c3116ed1daedec17be562d8e58-1 {}] regionserver.HStore(327): Store=585671c3116ed1daedec17be562d8e58/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T14:52:33,227 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 585671c3116ed1daedec17be562d8e58 2024-11-14T14:52:33,228 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34445/hbase/data/default/testReplayEditsWrittenViaHRegion/585671c3116ed1daedec17be562d8e58 2024-11-14T14:52:33,229 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:34445/hbase/data/default/testReplayEditsWrittenViaHRegion/585671c3116ed1daedec17be562d8e58 2024-11-14T14:52:33,229 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:34445/hbase/data/default/testReplayEditsWrittenViaHRegion/585671c3116ed1daedec17be562d8e58/recovered.edits/0000000000000000066 2024-11-14T14:52:33,255 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-14T14:52:33,259 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 30, skipped 0, firstSequenceIdInLog=37, maxSequenceIdInLog=66, path=hdfs://localhost:34445/hbase/data/default/testReplayEditsWrittenViaHRegion/585671c3116ed1daedec17be562d8e58/recovered.edits/0000000000000000066 2024-11-14T14:52:33,259 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 585671c3116ed1daedec17be562d8e58 3/3 column families, dataSize=2.55 KB heapSize=5.44 KB 2024-11-14T14:52:33,273 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/hbase/data/default/testReplayEditsWrittenViaHRegion/585671c3116ed1daedec17be562d8e58/.tmp/a/394b617f0e1b4a769455558db1e2f489 is 91, key is testReplayEditsWrittenViaHRegion/a:y0/1731595949036/Put/seqid=0 2024-11-14T14:52:33,279 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42819 is added to blk_1073741935_1115 (size=5958) 2024-11-14T14:52:33,279 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36189 is added to blk_1073741935_1115 (size=5958) 2024-11-14T14:52:33,279 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44549 is added to blk_1073741935_1115 (size=5958) 2024-11-14T14:52:33,280 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:34445/hbase/data/default/testReplayEditsWrittenViaHRegion/585671c3116ed1daedec17be562d8e58/.tmp/a/394b617f0e1b4a769455558db1e2f489 2024-11-14T14:52:33,297 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/hbase/data/default/testReplayEditsWrittenViaHRegion/585671c3116ed1daedec17be562d8e58/.tmp/b/a23492290c954ed79c59d4782bca5bdc is 91, key is testReplayEditsWrittenViaHRegion/b:y0/1731595949042/Put/seqid=0 2024-11-14T14:52:33,303 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36189 is added to blk_1073741936_1116 (size=5958) 2024-11-14T14:52:33,303 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44549 is added to blk_1073741936_1116 (size=5958) 2024-11-14T14:52:33,303 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42819 is added to blk_1073741936_1116 (size=5958) 2024-11-14T14:52:33,304 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:34445/hbase/data/default/testReplayEditsWrittenViaHRegion/585671c3116ed1daedec17be562d8e58/.tmp/b/a23492290c954ed79c59d4782bca5bdc 2024-11-14T14:52:33,322 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/hbase/data/default/testReplayEditsWrittenViaHRegion/585671c3116ed1daedec17be562d8e58/.tmp/c/e4a55268a3d9467db0d8a8e3a5158602 is 91, key is testReplayEditsWrittenViaHRegion/c:y0/1731595949050/Put/seqid=0 2024-11-14T14:52:33,327 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42819 is added to blk_1073741937_1117 (size=5958) 2024-11-14T14:52:33,327 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44549 is added to blk_1073741937_1117 (size=5958) 2024-11-14T14:52:33,328 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:34445/hbase/data/default/testReplayEditsWrittenViaHRegion/585671c3116ed1daedec17be562d8e58/.tmp/c/e4a55268a3d9467db0d8a8e3a5158602 2024-11-14T14:52:33,328 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36189 is added to blk_1073741937_1117 (size=5958) 2024-11-14T14:52:33,332 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/hbase/data/default/testReplayEditsWrittenViaHRegion/585671c3116ed1daedec17be562d8e58/.tmp/a/394b617f0e1b4a769455558db1e2f489 as hdfs://localhost:34445/hbase/data/default/testReplayEditsWrittenViaHRegion/585671c3116ed1daedec17be562d8e58/a/394b617f0e1b4a769455558db1e2f489 2024-11-14T14:52:33,336 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/hbase/data/default/testReplayEditsWrittenViaHRegion/585671c3116ed1daedec17be562d8e58/a/394b617f0e1b4a769455558db1e2f489, entries=10, sequenceid=66, filesize=5.8 K 2024-11-14T14:52:33,337 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/hbase/data/default/testReplayEditsWrittenViaHRegion/585671c3116ed1daedec17be562d8e58/.tmp/b/a23492290c954ed79c59d4782bca5bdc as hdfs://localhost:34445/hbase/data/default/testReplayEditsWrittenViaHRegion/585671c3116ed1daedec17be562d8e58/b/a23492290c954ed79c59d4782bca5bdc 2024-11-14T14:52:33,342 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/hbase/data/default/testReplayEditsWrittenViaHRegion/585671c3116ed1daedec17be562d8e58/b/a23492290c954ed79c59d4782bca5bdc, entries=10, sequenceid=66, filesize=5.8 K 2024-11-14T14:52:33,342 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/hbase/data/default/testReplayEditsWrittenViaHRegion/585671c3116ed1daedec17be562d8e58/.tmp/c/e4a55268a3d9467db0d8a8e3a5158602 as hdfs://localhost:34445/hbase/data/default/testReplayEditsWrittenViaHRegion/585671c3116ed1daedec17be562d8e58/c/e4a55268a3d9467db0d8a8e3a5158602 2024-11-14T14:52:33,346 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/hbase/data/default/testReplayEditsWrittenViaHRegion/585671c3116ed1daedec17be562d8e58/c/e4a55268a3d9467db0d8a8e3a5158602, entries=10, sequenceid=66, filesize=5.8 K 2024-11-14T14:52:33,347 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~2.55 KB/2610, heapSize ~5.39 KB/5520, currentSize=0 B/0 for 585671c3116ed1daedec17be562d8e58 in 87ms, sequenceid=66, compaction requested=false; wal=null 2024-11-14T14:52:33,347 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:34445/hbase/data/default/testReplayEditsWrittenViaHRegion/585671c3116ed1daedec17be562d8e58/recovered.edits/0000000000000000066 2024-11-14T14:52:33,348 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 585671c3116ed1daedec17be562d8e58 2024-11-14T14:52:33,348 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 585671c3116ed1daedec17be562d8e58 2024-11-14T14:52:33,349 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-14T14:52:33,350 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 585671c3116ed1daedec17be562d8e58 2024-11-14T14:52:33,352 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34445/hbase/data/default/testReplayEditsWrittenViaHRegion/585671c3116ed1daedec17be562d8e58/recovered.edits/66.seqid, newMaxSeqId=66, maxSeqId=35 2024-11-14T14:52:33,353 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 585671c3116ed1daedec17be562d8e58; next sequenceid=67; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63449077, jitterRate=-0.05453507602214813}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-14T14:52:33,353 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 585671c3116ed1daedec17be562d8e58: Writing region info on filesystem at 1731595953210Initializing all the Stores at 1731595953211 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731595953211Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731595953211Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731595953211Obtaining lock to block concurrent updates at 1731595953259 (+48 ms)Preparing flush snapshotting stores in 585671c3116ed1daedec17be562d8e58 at 1731595953259Finished memstore snapshotting testReplayEditsWrittenViaHRegion,,1731595948574.585671c3116ed1daedec17be562d8e58., syncing WAL and waiting on mvcc, flushsize=dataSize=2610, getHeapSize=5520, getOffHeapSize=0, getCellsCount=30 at 1731595953260 (+1 ms)Flushing stores of testReplayEditsWrittenViaHRegion,,1731595948574.585671c3116ed1daedec17be562d8e58. at 1731595953260Flushing 585671c3116ed1daedec17be562d8e58/a: creating writer at 1731595953260Flushing 585671c3116ed1daedec17be562d8e58/a: appending metadata at 1731595953272 (+12 ms)Flushing 585671c3116ed1daedec17be562d8e58/a: closing flushed file at 1731595953272Flushing 585671c3116ed1daedec17be562d8e58/b: creating writer at 1731595953284 (+12 ms)Flushing 585671c3116ed1daedec17be562d8e58/b: appending metadata at 1731595953297 (+13 ms)Flushing 585671c3116ed1daedec17be562d8e58/b: closing flushed file at 1731595953297Flushing 585671c3116ed1daedec17be562d8e58/c: creating writer at 1731595953308 (+11 ms)Flushing 585671c3116ed1daedec17be562d8e58/c: appending metadata at 1731595953321 (+13 ms)Flushing 585671c3116ed1daedec17be562d8e58/c: closing flushed file at 1731595953321Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1586850e: reopening flushed file at 1731595953332 (+11 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5e3f6c6b: reopening flushed file at 1731595953337 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1f61ee3c: reopening flushed file at 1731595953342 (+5 ms)Finished flush of dataSize ~2.55 KB/2610, heapSize ~5.39 KB/5520, currentSize=0 B/0 for 585671c3116ed1daedec17be562d8e58 in 87ms, sequenceid=66, compaction requested=false; wal=null at 1731595953347 (+5 ms)Cleaning up temporary data from old regions at 1731595953348 (+1 ms)Region opened successfully at 1731595953353 (+5 ms) 2024-11-14T14:52:33,366 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 585671c3116ed1daedec17be562d8e58, disabling compactions & flushes 2024-11-14T14:52:33,366 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1731595948574.585671c3116ed1daedec17be562d8e58. 2024-11-14T14:52:33,366 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1731595948574.585671c3116ed1daedec17be562d8e58. 2024-11-14T14:52:33,366 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1731595948574.585671c3116ed1daedec17be562d8e58. after waiting 0 ms 2024-11-14T14:52:33,366 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1731595948574.585671c3116ed1daedec17be562d8e58. 2024-11-14T14:52:33,368 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1731595948574.585671c3116ed1daedec17be562d8e58. 2024-11-14T14:52:33,368 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 585671c3116ed1daedec17be562d8e58: Waiting for close lock at 1731595953366Disabling compacts and flushes for region at 1731595953366Disabling writes for close at 1731595953366Writing region close event to WAL at 1731595953368 (+2 ms)Closed at 1731595953368 2024-11-14T14:52:33,370 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44549 is added to blk_1073741934_1114 (size=175) 2024-11-14T14:52:33,370 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36189 is added to blk_1073741934_1114 (size=175) 2024-11-14T14:52:33,370 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42819 is added to blk_1073741934_1114 (size=175) 2024-11-14T14:52:33,372 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-14T14:52:33,372 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1731595953168) 2024-11-14T14:52:33,389 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsWrittenViaHRegion Thread=448 (was 444) Potentially hanging thread: IPC Client (1319272049) connection to localhost/127.0.0.1:34445 from jenkinstestReplayEditsWrittenViaHRegion java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_978909817_22 at /127.0.0.1:33818 [Waiting for operation #8] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_978909817_22 at /127.0.0.1:45748 [Waiting for operation #22] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-28-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_978909817_22 at /127.0.0.1:34144 [Waiting for operation #13] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-28-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-28-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkinstestReplayEditsWrittenViaHRegion@localhost:34445 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1416 (was 1350) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=141 (was 153), ProcessCount=11 (was 11), AvailableMemoryMB=7507 (was 7520) 2024-11-14T14:52:33,389 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1416 is superior to 1024 2024-11-14T14:52:33,389 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-14T14:52:33,390 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-14T14:52:33,390 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-14T14:52:33,390 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T14:52:33,390 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T14:52:33,391 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T14:52:33,391 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-14T14:52:33,391 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=223486682, stopped=false 2024-11-14T14:52:33,391 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=da1c6afcd1f9,42151,1731595912734 2024-11-14T14:52:33,393 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46007-0x101a74438830003, quorum=127.0.0.1:51231, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-14T14:52:33,393 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37695-0x101a74438830001, quorum=127.0.0.1:51231, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-14T14:52:33,393 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42151-0x101a74438830000, quorum=127.0.0.1:51231, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-14T14:52:33,393 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37695-0x101a74438830001, quorum=127.0.0.1:51231, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T14:52:33,393 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46007-0x101a74438830003, quorum=127.0.0.1:51231, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T14:52:33,393 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42151-0x101a74438830000, quorum=127.0.0.1:51231, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T14:52:33,394 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-14T14:52:33,394 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:46007-0x101a74438830003, quorum=127.0.0.1:51231, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-14T14:52:33,394 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:42151-0x101a74438830000, quorum=127.0.0.1:51231, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-14T14:52:33,394 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-14T14:52:33,394 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-14T14:52:33,394 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T14:52:33,395 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:37695-0x101a74438830001, quorum=127.0.0.1:51231, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-14T14:52:33,395 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'da1c6afcd1f9,37695,1731595913427' ***** 2024-11-14T14:52:33,395 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-14T14:52:33,395 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'da1c6afcd1f9,46007,1731595913586' ***** 2024-11-14T14:52:33,395 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-14T14:52:33,395 INFO [RS:0;da1c6afcd1f9:37695 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-14T14:52:33,395 INFO [RS:2;da1c6afcd1f9:46007 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-14T14:52:33,396 INFO [RS:0;da1c6afcd1f9:37695 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-14T14:52:33,396 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-14T14:52:33,396 INFO [RS:0;da1c6afcd1f9:37695 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-14T14:52:33,396 INFO [RS:2;da1c6afcd1f9:46007 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-14T14:52:33,396 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-14T14:52:33,396 INFO [RS:0;da1c6afcd1f9:37695 {}] regionserver.HRegionServer(3091): Received CLOSE for cf7ad4aad0d0e3c5a9f31c16500a8e87 2024-11-14T14:52:33,396 INFO [RS:2;da1c6afcd1f9:46007 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-14T14:52:33,396 INFO [RS:2;da1c6afcd1f9:46007 {}] regionserver.HRegionServer(959): stopping server da1c6afcd1f9,46007,1731595913586 2024-11-14T14:52:33,396 INFO [RS:0;da1c6afcd1f9:37695 {}] regionserver.HRegionServer(959): stopping server da1c6afcd1f9,37695,1731595913427 2024-11-14T14:52:33,396 INFO [RS:2;da1c6afcd1f9:46007 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-14T14:52:33,396 INFO [RS:0;da1c6afcd1f9:37695 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-14T14:52:33,396 INFO [RS:2;da1c6afcd1f9:46007 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:2;da1c6afcd1f9:46007. 2024-11-14T14:52:33,396 INFO [RS:0;da1c6afcd1f9:37695 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;da1c6afcd1f9:37695. 2024-11-14T14:52:33,396 DEBUG [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing cf7ad4aad0d0e3c5a9f31c16500a8e87, disabling compactions & flushes 2024-11-14T14:52:33,396 DEBUG [RS:2;da1c6afcd1f9:46007 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-14T14:52:33,396 DEBUG [RS:2;da1c6afcd1f9:46007 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T14:52:33,396 DEBUG [RS:0;da1c6afcd1f9:37695 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-14T14:52:33,396 INFO [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1731595929525.cf7ad4aad0d0e3c5a9f31c16500a8e87. 2024-11-14T14:52:33,396 DEBUG [RS:0;da1c6afcd1f9:37695 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T14:52:33,396 DEBUG [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731595929525.cf7ad4aad0d0e3c5a9f31c16500a8e87. 2024-11-14T14:52:33,396 INFO [RS:2;da1c6afcd1f9:46007 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-14T14:52:33,396 INFO [RS:2;da1c6afcd1f9:46007 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-14T14:52:33,396 DEBUG [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731595929525.cf7ad4aad0d0e3c5a9f31c16500a8e87. after waiting 0 ms 2024-11-14T14:52:33,396 INFO [RS:2;da1c6afcd1f9:46007 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-14T14:52:33,397 INFO [RS:0;da1c6afcd1f9:37695 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-14T14:52:33,397 DEBUG [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1731595929525.cf7ad4aad0d0e3c5a9f31c16500a8e87. 2024-11-14T14:52:33,397 DEBUG [RS:0;da1c6afcd1f9:37695 {}] regionserver.HRegionServer(1325): Online Regions={cf7ad4aad0d0e3c5a9f31c16500a8e87=testReplayEditsAfterRegionMovedWithMultiCF,,1731595929525.cf7ad4aad0d0e3c5a9f31c16500a8e87.} 2024-11-14T14:52:33,397 INFO [RS:2;da1c6afcd1f9:46007 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-14T14:52:33,397 DEBUG [RS:0;da1c6afcd1f9:37695 {}] regionserver.HRegionServer(1351): Waiting on cf7ad4aad0d0e3c5a9f31c16500a8e87 2024-11-14T14:52:33,397 INFO [RS:2;da1c6afcd1f9:46007 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-14T14:52:33,397 DEBUG [RS:2;da1c6afcd1f9:46007 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-14T14:52:33,397 DEBUG [RS:2;da1c6afcd1f9:46007 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-14T14:52:33,397 DEBUG [RS_CLOSE_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-14T14:52:33,397 INFO [RS_CLOSE_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-14T14:52:33,397 DEBUG [RS_CLOSE_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-14T14:52:33,397 DEBUG [RS_CLOSE_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-14T14:52:33,397 DEBUG [RS_CLOSE_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-14T14:52:33,397 INFO [RS_CLOSE_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=6.86 KB heapSize=11.45 KB 2024-11-14T14:52:33,401 DEBUG [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34445/user/jenkins/test-data/35161b07-018a-0344-8c43-1519f3812567/data/default/testReplayEditsAfterRegionMovedWithMultiCF/cf7ad4aad0d0e3c5a9f31c16500a8e87/recovered.edits/20.seqid, newMaxSeqId=20, maxSeqId=17 2024-11-14T14:52:33,402 INFO [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1731595929525.cf7ad4aad0d0e3c5a9f31c16500a8e87. 2024-11-14T14:52:33,402 DEBUG [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for cf7ad4aad0d0e3c5a9f31c16500a8e87: Waiting for close lock at 1731595953396Running coprocessor pre-close hooks at 1731595953396Disabling compacts and flushes for region at 1731595953396Disabling writes for close at 1731595953396Writing region close event to WAL at 1731595953398 (+2 ms)Running coprocessor post-close hooks at 1731595953402 (+4 ms)Closed at 1731595953402 2024-11-14T14:52:33,402 DEBUG [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1731595929525.cf7ad4aad0d0e3c5a9f31c16500a8e87. 2024-11-14T14:52:33,419 DEBUG [RS_CLOSE_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/35161b07-018a-0344-8c43-1519f3812567/data/hbase/meta/1588230740/.tmp/info/cf3bde2a58b74e4c8b9d6f14c8709bc4 is 205, key is testReplayEditsAfterRegionMovedWithMultiCF,,1731595929525.cf7ad4aad0d0e3c5a9f31c16500a8e87./info:regioninfo/1731595932504/Put/seqid=0 2024-11-14T14:52:33,425 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42819 is added to blk_1073741938_1118 (size=8243) 2024-11-14T14:52:33,425 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44549 is added to blk_1073741938_1118 (size=8243) 2024-11-14T14:52:33,426 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36189 is added to blk_1073741938_1118 (size=8243) 2024-11-14T14:52:33,426 INFO [RS_CLOSE_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.65 KB at sequenceid=21 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/35161b07-018a-0344-8c43-1519f3812567/data/hbase/meta/1588230740/.tmp/info/cf3bde2a58b74e4c8b9d6f14c8709bc4 2024-11-14T14:52:33,445 DEBUG [RS_CLOSE_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/35161b07-018a-0344-8c43-1519f3812567/data/hbase/meta/1588230740/.tmp/ns/b36975da1a1f471b97372c33ee9a376c is 43, key is default/ns:d/1731595915658/Put/seqid=0 2024-11-14T14:52:33,450 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44549 is added to blk_1073741939_1119 (size=5153) 2024-11-14T14:52:33,451 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42819 is added to blk_1073741939_1119 (size=5153) 2024-11-14T14:52:33,451 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36189 is added to blk_1073741939_1119 (size=5153) 2024-11-14T14:52:33,452 INFO [RS_CLOSE_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=21 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/35161b07-018a-0344-8c43-1519f3812567/data/hbase/meta/1588230740/.tmp/ns/b36975da1a1f471b97372c33ee9a376c 2024-11-14T14:52:33,470 DEBUG [RS_CLOSE_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/35161b07-018a-0344-8c43-1519f3812567/data/hbase/meta/1588230740/.tmp/table/066c0a78bd7846f2999742eb8f07d1b4 is 78, key is testReplayEditsAfterRegionMovedWithMultiCF/table:state/1731595929940/Put/seqid=0 2024-11-14T14:52:33,475 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42819 is added to blk_1073741940_1120 (size=5431) 2024-11-14T14:52:33,475 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44549 is added to blk_1073741940_1120 (size=5431) 2024-11-14T14:52:33,476 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36189 is added to blk_1073741940_1120 (size=5431) 2024-11-14T14:52:33,476 INFO [RS_CLOSE_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=148 B at sequenceid=21 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/35161b07-018a-0344-8c43-1519f3812567/data/hbase/meta/1588230740/.tmp/table/066c0a78bd7846f2999742eb8f07d1b4 2024-11-14T14:52:33,481 DEBUG [RS_CLOSE_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/35161b07-018a-0344-8c43-1519f3812567/data/hbase/meta/1588230740/.tmp/info/cf3bde2a58b74e4c8b9d6f14c8709bc4 as hdfs://localhost:34445/user/jenkins/test-data/35161b07-018a-0344-8c43-1519f3812567/data/hbase/meta/1588230740/info/cf3bde2a58b74e4c8b9d6f14c8709bc4 2024-11-14T14:52:33,484 INFO [regionserver/da1c6afcd1f9:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-14T14:52:33,485 INFO [regionserver/da1c6afcd1f9:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-14T14:52:33,486 INFO [RS_CLOSE_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/35161b07-018a-0344-8c43-1519f3812567/data/hbase/meta/1588230740/info/cf3bde2a58b74e4c8b9d6f14c8709bc4, entries=18, sequenceid=21, filesize=8.0 K 2024-11-14T14:52:33,487 DEBUG [RS_CLOSE_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/35161b07-018a-0344-8c43-1519f3812567/data/hbase/meta/1588230740/.tmp/ns/b36975da1a1f471b97372c33ee9a376c as hdfs://localhost:34445/user/jenkins/test-data/35161b07-018a-0344-8c43-1519f3812567/data/hbase/meta/1588230740/ns/b36975da1a1f471b97372c33ee9a376c 2024-11-14T14:52:33,492 INFO [RS_CLOSE_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/35161b07-018a-0344-8c43-1519f3812567/data/hbase/meta/1588230740/ns/b36975da1a1f471b97372c33ee9a376c, entries=2, sequenceid=21, filesize=5.0 K 2024-11-14T14:52:33,492 DEBUG [RS_CLOSE_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/35161b07-018a-0344-8c43-1519f3812567/data/hbase/meta/1588230740/.tmp/table/066c0a78bd7846f2999742eb8f07d1b4 as hdfs://localhost:34445/user/jenkins/test-data/35161b07-018a-0344-8c43-1519f3812567/data/hbase/meta/1588230740/table/066c0a78bd7846f2999742eb8f07d1b4 2024-11-14T14:52:33,497 INFO [RS_CLOSE_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/35161b07-018a-0344-8c43-1519f3812567/data/hbase/meta/1588230740/table/066c0a78bd7846f2999742eb8f07d1b4, entries=2, sequenceid=21, filesize=5.3 K 2024-11-14T14:52:33,498 INFO [RS_CLOSE_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~6.86 KB/7029, heapSize ~11.16 KB/11424, currentSize=0 B/0 for 1588230740 in 100ms, sequenceid=21, compaction requested=false 2024-11-14T14:52:33,501 DEBUG [RS_CLOSE_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34445/user/jenkins/test-data/35161b07-018a-0344-8c43-1519f3812567/data/hbase/meta/1588230740/recovered.edits/24.seqid, newMaxSeqId=24, maxSeqId=1 2024-11-14T14:52:33,502 DEBUG [RS_CLOSE_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-14T14:52:33,502 INFO [RS_CLOSE_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-14T14:52:33,502 DEBUG [RS_CLOSE_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731595953397Running coprocessor pre-close hooks at 1731595953397Disabling compacts and flushes for region at 1731595953397Disabling writes for close at 1731595953397Obtaining lock to block concurrent updates at 1731595953397Preparing flush snapshotting stores in 1588230740 at 1731595953397Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=7029, getHeapSize=11664, getOffHeapSize=0, getCellsCount=48 at 1731595953398 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1731595953399 (+1 ms)Flushing 1588230740/info: creating writer at 1731595953399Flushing 1588230740/info: appending metadata at 1731595953419 (+20 ms)Flushing 1588230740/info: closing flushed file at 1731595953419Flushing 1588230740/ns: creating writer at 1731595953431 (+12 ms)Flushing 1588230740/ns: appending metadata at 1731595953445 (+14 ms)Flushing 1588230740/ns: closing flushed file at 1731595953445Flushing 1588230740/table: creating writer at 1731595953456 (+11 ms)Flushing 1588230740/table: appending metadata at 1731595953470 (+14 ms)Flushing 1588230740/table: closing flushed file at 1731595953470Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4e1832e6: reopening flushed file at 1731595953481 (+11 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@538b5868: reopening flushed file at 1731595953487 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@38101109: reopening flushed file at 1731595953492 (+5 ms)Finished flush of dataSize ~6.86 KB/7029, heapSize ~11.16 KB/11424, currentSize=0 B/0 for 1588230740 in 100ms, sequenceid=21, compaction requested=false at 1731595953498 (+6 ms)Writing region close event to WAL at 1731595953499 (+1 ms)Running coprocessor post-close hooks at 1731595953502 (+3 ms)Closed at 1731595953502 2024-11-14T14:52:33,502 DEBUG [RS_CLOSE_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-14T14:52:33,597 INFO [RS:0;da1c6afcd1f9:37695 {}] regionserver.HRegionServer(976): stopping server da1c6afcd1f9,37695,1731595913427; all regions closed. 2024-11-14T14:52:33,597 INFO [RS:2;da1c6afcd1f9:46007 {}] regionserver.HRegionServer(976): stopping server da1c6afcd1f9,46007,1731595913586; all regions closed. 2024-11-14T14:52:33,599 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42819 is added to blk_1073741834_1010 (size=3621) 2024-11-14T14:52:33,600 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44549 is added to blk_1073741834_1010 (size=3621) 2024-11-14T14:52:33,600 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44549 is added to blk_1073741836_1012 (size=9487) 2024-11-14T14:52:33,600 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36189 is added to blk_1073741834_1010 (size=3621) 2024-11-14T14:52:33,600 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42819 is added to blk_1073741836_1012 (size=9487) 2024-11-14T14:52:33,600 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36189 is added to blk_1073741836_1012 (size=9487) 2024-11-14T14:52:33,602 DEBUG [RS:2;da1c6afcd1f9:46007 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/35161b07-018a-0344-8c43-1519f3812567/oldWALs 2024-11-14T14:52:33,602 INFO [RS:2;da1c6afcd1f9:46007 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL da1c6afcd1f9%2C46007%2C1731595913586.meta:.meta(num 1731595915439) 2024-11-14T14:52:33,602 DEBUG [RS:0;da1c6afcd1f9:37695 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/35161b07-018a-0344-8c43-1519f3812567/oldWALs 2024-11-14T14:52:33,603 INFO [RS:0;da1c6afcd1f9:37695 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL da1c6afcd1f9%2C37695%2C1731595913427:(num 1731595915018) 2024-11-14T14:52:33,603 DEBUG [RS:0;da1c6afcd1f9:37695 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T14:52:33,603 INFO [RS:0;da1c6afcd1f9:37695 {}] regionserver.LeaseManager(133): Closed leases 2024-11-14T14:52:33,603 INFO [RS:0;da1c6afcd1f9:37695 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-14T14:52:33,603 INFO [RS:0;da1c6afcd1f9:37695 {}] hbase.ChoreService(370): Chore service for: regionserver/da1c6afcd1f9:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-14T14:52:33,603 INFO [RS:0;da1c6afcd1f9:37695 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-14T14:52:33,603 INFO [RS:0;da1c6afcd1f9:37695 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-14T14:52:33,603 INFO [regionserver/da1c6afcd1f9:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-14T14:52:33,603 INFO [RS:0;da1c6afcd1f9:37695 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-14T14:52:33,603 INFO [RS:0;da1c6afcd1f9:37695 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-14T14:52:33,604 INFO [RS:0;da1c6afcd1f9:37695 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:37695 2024-11-14T14:52:33,604 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42819 is added to blk_1073741833_1009 (size=175) 2024-11-14T14:52:33,604 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44549 is added to blk_1073741833_1009 (size=175) 2024-11-14T14:52:33,604 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36189 is added to blk_1073741833_1009 (size=175) 2024-11-14T14:52:33,606 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37695-0x101a74438830001, quorum=127.0.0.1:51231, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/da1c6afcd1f9,37695,1731595913427 2024-11-14T14:52:33,606 INFO [RS:0;da1c6afcd1f9:37695 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-14T14:52:33,606 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42151-0x101a74438830000, quorum=127.0.0.1:51231, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-14T14:52:33,608 DEBUG [RS:2;da1c6afcd1f9:46007 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/35161b07-018a-0344-8c43-1519f3812567/oldWALs 2024-11-14T14:52:33,608 INFO [RS:2;da1c6afcd1f9:46007 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL da1c6afcd1f9%2C46007%2C1731595913586:(num 1731595915018) 2024-11-14T14:52:33,608 DEBUG [RS:2;da1c6afcd1f9:46007 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T14:52:33,608 INFO [RS:2;da1c6afcd1f9:46007 {}] regionserver.LeaseManager(133): Closed leases 2024-11-14T14:52:33,608 INFO [RS:2;da1c6afcd1f9:46007 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-14T14:52:33,608 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [da1c6afcd1f9,37695,1731595913427] 2024-11-14T14:52:33,608 INFO [RS:2;da1c6afcd1f9:46007 {}] hbase.ChoreService(370): Chore service for: regionserver/da1c6afcd1f9:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-14T14:52:33,608 INFO [RS:2;da1c6afcd1f9:46007 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-14T14:52:33,608 INFO [regionserver/da1c6afcd1f9:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-14T14:52:33,608 INFO [RS:2;da1c6afcd1f9:46007 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:46007 2024-11-14T14:52:33,610 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/da1c6afcd1f9,37695,1731595913427 already deleted, retry=false 2024-11-14T14:52:33,610 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; da1c6afcd1f9,37695,1731595913427 expired; onlineServers=1 2024-11-14T14:52:33,610 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46007-0x101a74438830003, quorum=127.0.0.1:51231, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/da1c6afcd1f9,46007,1731595913586 2024-11-14T14:52:33,610 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42151-0x101a74438830000, quorum=127.0.0.1:51231, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-14T14:52:33,610 INFO [RS:2;da1c6afcd1f9:46007 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-14T14:52:33,612 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [da1c6afcd1f9,46007,1731595913586] 2024-11-14T14:52:33,613 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/da1c6afcd1f9,46007,1731595913586 already deleted, retry=false 2024-11-14T14:52:33,613 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; da1c6afcd1f9,46007,1731595913586 expired; onlineServers=0 2024-11-14T14:52:33,613 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'da1c6afcd1f9,42151,1731595912734' ***** 2024-11-14T14:52:33,613 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-14T14:52:33,613 INFO [M:0;da1c6afcd1f9:42151 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-14T14:52:33,613 INFO [M:0;da1c6afcd1f9:42151 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-14T14:52:33,613 DEBUG [M:0;da1c6afcd1f9:42151 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-14T14:52:33,614 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-14T14:52:33,614 DEBUG [M:0;da1c6afcd1f9:42151 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-14T14:52:33,614 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster-HFileCleaner.large.0-1731595914676 {}] cleaner.HFileCleaner(306): Exit Thread[master/da1c6afcd1f9:0:becomeActiveMaster-HFileCleaner.large.0-1731595914676,5,FailOnTimeoutGroup] 2024-11-14T14:52:33,614 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster-HFileCleaner.small.0-1731595914677 {}] cleaner.HFileCleaner(306): Exit Thread[master/da1c6afcd1f9:0:becomeActiveMaster-HFileCleaner.small.0-1731595914677,5,FailOnTimeoutGroup] 2024-11-14T14:52:33,614 INFO [M:0;da1c6afcd1f9:42151 {}] hbase.ChoreService(370): Chore service for: master/da1c6afcd1f9:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-14T14:52:33,614 INFO [M:0;da1c6afcd1f9:42151 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-14T14:52:33,614 DEBUG [M:0;da1c6afcd1f9:42151 {}] master.HMaster(1795): Stopping service threads 2024-11-14T14:52:33,614 INFO [M:0;da1c6afcd1f9:42151 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-14T14:52:33,614 INFO [M:0;da1c6afcd1f9:42151 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-14T14:52:33,615 INFO [M:0;da1c6afcd1f9:42151 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-14T14:52:33,615 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42151-0x101a74438830000, quorum=127.0.0.1:51231, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-14T14:52:33,615 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-14T14:52:33,615 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42151-0x101a74438830000, quorum=127.0.0.1:51231, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T14:52:33,615 DEBUG [M:0;da1c6afcd1f9:42151 {}] zookeeper.ZKUtil(347): master:42151-0x101a74438830000, quorum=127.0.0.1:51231, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-14T14:52:33,616 WARN [M:0;da1c6afcd1f9:42151 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-14T14:52:33,616 INFO [M:0;da1c6afcd1f9:42151 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:34445/user/jenkins/test-data/35161b07-018a-0344-8c43-1519f3812567/.lastflushedseqids 2024-11-14T14:52:33,627 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36189 is added to blk_1073741941_1121 (size=138) 2024-11-14T14:52:33,628 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44549 is added to blk_1073741941_1121 (size=138) 2024-11-14T14:52:33,628 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42819 is added to blk_1073741941_1121 (size=138) 2024-11-14T14:52:33,628 INFO [M:0;da1c6afcd1f9:42151 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-14T14:52:33,628 INFO [M:0;da1c6afcd1f9:42151 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-14T14:52:33,628 DEBUG [M:0;da1c6afcd1f9:42151 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-14T14:52:33,629 INFO [M:0;da1c6afcd1f9:42151 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-14T14:52:33,629 DEBUG [M:0;da1c6afcd1f9:42151 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-14T14:52:33,629 DEBUG [M:0;da1c6afcd1f9:42151 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-14T14:52:33,629 DEBUG [M:0;da1c6afcd1f9:42151 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-14T14:52:33,629 INFO [M:0;da1c6afcd1f9:42151 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=68.32 KB heapSize=83.72 KB 2024-11-14T14:52:33,644 DEBUG [M:0;da1c6afcd1f9:42151 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/35161b07-018a-0344-8c43-1519f3812567/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/005cb4264c52470081e32db3eede5376 is 82, key is hbase:meta,,1/info:regioninfo/1731595915553/Put/seqid=0 2024-11-14T14:52:33,649 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36189 is added to blk_1073741942_1122 (size=5672) 2024-11-14T14:52:33,649 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44549 is added to blk_1073741942_1122 (size=5672) 2024-11-14T14:52:33,649 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42819 is added to blk_1073741942_1122 (size=5672) 2024-11-14T14:52:33,650 INFO [M:0;da1c6afcd1f9:42151 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/35161b07-018a-0344-8c43-1519f3812567/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/005cb4264c52470081e32db3eede5376 2024-11-14T14:52:33,668 DEBUG [M:0;da1c6afcd1f9:42151 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/35161b07-018a-0344-8c43-1519f3812567/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/eb29e034de194057acbc051eb03618c2 is 1074, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1731595929945/Put/seqid=0 2024-11-14T14:52:33,676 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44549 is added to blk_1073741943_1123 (size=7753) 2024-11-14T14:52:33,677 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36189 is added to blk_1073741943_1123 (size=7753) 2024-11-14T14:52:33,677 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42819 is added to blk_1073741943_1123 (size=7753) 2024-11-14T14:52:33,677 INFO [M:0;da1c6afcd1f9:42151 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=67.58 KB at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/35161b07-018a-0344-8c43-1519f3812567/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/eb29e034de194057acbc051eb03618c2 2024-11-14T14:52:33,682 INFO [M:0;da1c6afcd1f9:42151 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for eb29e034de194057acbc051eb03618c2 2024-11-14T14:52:33,697 DEBUG [M:0;da1c6afcd1f9:42151 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/35161b07-018a-0344-8c43-1519f3812567/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/28d909fb9cd44f70937dccf0bede8ec3 is 69, key is da1c6afcd1f9,37695,1731595913427/rs:state/1731595914756/Put/seqid=0 2024-11-14T14:52:33,703 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36189 is added to blk_1073741944_1124 (size=5440) 2024-11-14T14:52:33,703 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44549 is added to blk_1073741944_1124 (size=5440) 2024-11-14T14:52:33,704 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42819 is added to blk_1073741944_1124 (size=5440) 2024-11-14T14:52:33,704 INFO [M:0;da1c6afcd1f9:42151 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=249 B at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/35161b07-018a-0344-8c43-1519f3812567/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/28d909fb9cd44f70937dccf0bede8ec3 2024-11-14T14:52:33,708 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37695-0x101a74438830001, quorum=127.0.0.1:51231, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-14T14:52:33,708 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37695-0x101a74438830001, quorum=127.0.0.1:51231, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-14T14:52:33,708 INFO [RS:0;da1c6afcd1f9:37695 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-14T14:52:33,708 INFO [RS:0;da1c6afcd1f9:37695 {}] regionserver.HRegionServer(1031): Exiting; stopping=da1c6afcd1f9,37695,1731595913427; zookeeper connection closed. 2024-11-14T14:52:33,709 INFO [M:0;da1c6afcd1f9:42151 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 28d909fb9cd44f70937dccf0bede8ec3 2024-11-14T14:52:33,709 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@631c3f0b {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@631c3f0b 2024-11-14T14:52:33,709 DEBUG [M:0;da1c6afcd1f9:42151 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/35161b07-018a-0344-8c43-1519f3812567/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/005cb4264c52470081e32db3eede5376 as hdfs://localhost:34445/user/jenkins/test-data/35161b07-018a-0344-8c43-1519f3812567/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/005cb4264c52470081e32db3eede5376 2024-11-14T14:52:33,712 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46007-0x101a74438830003, quorum=127.0.0.1:51231, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-14T14:52:33,712 INFO [RS:2;da1c6afcd1f9:46007 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-14T14:52:33,712 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46007-0x101a74438830003, quorum=127.0.0.1:51231, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-14T14:52:33,712 INFO [RS:2;da1c6afcd1f9:46007 {}] regionserver.HRegionServer(1031): Exiting; stopping=da1c6afcd1f9,46007,1731595913586; zookeeper connection closed. 2024-11-14T14:52:33,712 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@55da2680 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@55da2680 2024-11-14T14:52:33,713 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 3 regionserver(s) complete 2024-11-14T14:52:33,714 INFO [M:0;da1c6afcd1f9:42151 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/35161b07-018a-0344-8c43-1519f3812567/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/005cb4264c52470081e32db3eede5376, entries=8, sequenceid=168, filesize=5.5 K 2024-11-14T14:52:33,715 DEBUG [M:0;da1c6afcd1f9:42151 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/35161b07-018a-0344-8c43-1519f3812567/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/eb29e034de194057acbc051eb03618c2 as hdfs://localhost:34445/user/jenkins/test-data/35161b07-018a-0344-8c43-1519f3812567/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/eb29e034de194057acbc051eb03618c2 2024-11-14T14:52:33,720 INFO [M:0;da1c6afcd1f9:42151 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for eb29e034de194057acbc051eb03618c2 2024-11-14T14:52:33,720 INFO [M:0;da1c6afcd1f9:42151 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/35161b07-018a-0344-8c43-1519f3812567/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/eb29e034de194057acbc051eb03618c2, entries=17, sequenceid=168, filesize=7.6 K 2024-11-14T14:52:33,721 DEBUG [M:0;da1c6afcd1f9:42151 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/35161b07-018a-0344-8c43-1519f3812567/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/28d909fb9cd44f70937dccf0bede8ec3 as hdfs://localhost:34445/user/jenkins/test-data/35161b07-018a-0344-8c43-1519f3812567/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/28d909fb9cd44f70937dccf0bede8ec3 2024-11-14T14:52:33,725 INFO [M:0;da1c6afcd1f9:42151 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 28d909fb9cd44f70937dccf0bede8ec3 2024-11-14T14:52:33,725 INFO [M:0;da1c6afcd1f9:42151 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/35161b07-018a-0344-8c43-1519f3812567/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/28d909fb9cd44f70937dccf0bede8ec3, entries=3, sequenceid=168, filesize=5.3 K 2024-11-14T14:52:33,726 INFO [M:0;da1c6afcd1f9:42151 {}] regionserver.HRegion(3140): Finished flush of dataSize ~68.32 KB/69960, heapSize ~83.42 KB/85424, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 97ms, sequenceid=168, compaction requested=false 2024-11-14T14:52:33,727 INFO [M:0;da1c6afcd1f9:42151 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-14T14:52:33,727 DEBUG [M:0;da1c6afcd1f9:42151 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731595953628Disabling compacts and flushes for region at 1731595953628Disabling writes for close at 1731595953629 (+1 ms)Obtaining lock to block concurrent updates at 1731595953629Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731595953629Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=69960, getHeapSize=85664, getOffHeapSize=0, getCellsCount=195 at 1731595953629Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731595953630 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731595953630Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731595953643 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731595953643Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731595953654 (+11 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731595953668 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731595953668Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731595953682 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731595953697 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731595953697Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3ae5bbe3: reopening flushed file at 1731595953709 (+12 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3af98928: reopening flushed file at 1731595953714 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3f396647: reopening flushed file at 1731595953720 (+6 ms)Finished flush of dataSize ~68.32 KB/69960, heapSize ~83.42 KB/85424, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 97ms, sequenceid=168, compaction requested=false at 1731595953726 (+6 ms)Writing region close event to WAL at 1731595953727 (+1 ms)Closed at 1731595953727 2024-11-14T14:52:33,729 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42819 is added to blk_1073741830_1006 (size=83582) 2024-11-14T14:52:33,729 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44549 is added to blk_1073741830_1006 (size=83582) 2024-11-14T14:52:33,730 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36189 is added to blk_1073741830_1006 (size=83582) 2024-11-14T14:52:33,730 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-14T14:52:33,730 INFO [M:0;da1c6afcd1f9:42151 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-14T14:52:33,730 INFO [M:0;da1c6afcd1f9:42151 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:42151 2024-11-14T14:52:33,731 INFO [M:0;da1c6afcd1f9:42151 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-14T14:52:33,833 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42151-0x101a74438830000, quorum=127.0.0.1:51231, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-14T14:52:33,833 INFO [M:0;da1c6afcd1f9:42151 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-14T14:52:33,833 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42151-0x101a74438830000, quorum=127.0.0.1:51231, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-14T14:52:33,838 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731595932881/wal.1731595933272 with renewLeaseKey: DEFAULT_16655 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731595932881/wal.1731595933272 (inode 16655) Holder DFSClient_NONMAPREDUCE_-1791616411_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731595932881/wal.1731595933272 (inode 16655) Holder DFSClient_NONMAPREDUCE_-1791616411_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy45.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy46.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 25 more 2024-11-14T14:52:33,839 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731595924996/wal.1731595925103 with renewLeaseKey: DEFAULT_16586 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:52:33,840 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731595948573/wal.1731595948873 with renewLeaseKey: DEFAULT_16767 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:52:33,842 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731595933478/wal.1731595942804 with renewLeaseKey: DEFAULT_16678 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731595933478/wal.1731595942804 (inode 16678) Holder DFSClient_NONMAPREDUCE_-1791616411_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731595933478/wal.1731595942804 (inode 16678) Holder DFSClient_NONMAPREDUCE_-1791616411_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy45.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy46.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 25 more 2024-11-14T14:52:33,842 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731595917323/wal.1731595917418 with renewLeaseKey: DEFAULT_16506 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:52:33,844 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testdatalosswheninputerror-manual,16010,1731595943050/wal.1731595943737 with renewLeaseKey: DEFAULT_16704 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testdatalosswheninputerror-manual,16010,1731595943050/wal.1731595943737 (inode 16704) Holder DFSClient_NONMAPREDUCE_-1791616411_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testdatalosswheninputerror-manual,16010,1731595943050/wal.1731595943737 (inode 16704) Holder DFSClient_NONMAPREDUCE_-1791616411_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy45.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy46.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 25 more 2024-11-14T14:52:33,845 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731595943987/wal.1731595944079 with renewLeaseKey: DEFAULT_16726 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:52:33,847 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testnameconflictwhensplit1-manual,16010,1731595916722/wal.1731595917131 with renewLeaseKey: DEFAULT_16485 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testnameconflictwhensplit1-manual,16010,1731595916722/wal.1731595917131 (inode 16485) Holder DFSClient_NONMAPREDUCE_-1791616411_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testnameconflictwhensplit1-manual,16010,1731595916722/wal.1731595917131 (inode 16485) Holder DFSClient_NONMAPREDUCE_-1791616411_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy45.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy46.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 25 more 2024-11-14T14:52:33,849 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testnameconflictwhensplit0-manual,16010,1731595915935/wal.1731595916407 with renewLeaseKey: DEFAULT_16462 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testnameconflictwhensplit0-manual,16010,1731595915935/wal.1731595916407 (inode 16462) Holder DFSClient_NONMAPREDUCE_-1791616411_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testnameconflictwhensplit0-manual,16010,1731595915935/wal.1731595916407 (inode 16462) Holder DFSClient_NONMAPREDUCE_-1791616411_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy45.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy46.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 25 more 2024-11-14T14:52:33,853 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@35f1150e{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-14T14:52:33,855 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@13a77e13{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-14T14:52:33,855 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-14T14:52:33,855 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@f9972d0{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-14T14:52:33,856 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@2a6d5e13{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c843459-ed19-f974-f761-d73dee21a91b/hadoop.log.dir/,STOPPED} 2024-11-14T14:52:33,858 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-14T14:52:33,858 WARN [BP-1055376130-172.17.0.2-1731595909704 heartbeating to localhost/127.0.0.1:34445 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-14T14:52:33,858 WARN [BP-1055376130-172.17.0.2-1731595909704 heartbeating to localhost/127.0.0.1:34445 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1055376130-172.17.0.2-1731595909704 (Datanode Uuid c57b7594-0e99-485a-938c-81b59bea0cf1) service to localhost/127.0.0.1:34445 2024-11-14T14:52:33,858 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-14T14:52:33,859 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c843459-ed19-f974-f761-d73dee21a91b/cluster_fceed6e6-d3b5-652e-82f8-dddc2c9b7e2b/data/data5/current/BP-1055376130-172.17.0.2-1731595909704 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-14T14:52:33,860 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c843459-ed19-f974-f761-d73dee21a91b/cluster_fceed6e6-d3b5-652e-82f8-dddc2c9b7e2b/data/data6/current/BP-1055376130-172.17.0.2-1731595909704 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-14T14:52:33,860 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-14T14:52:33,862 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@7bd427b8{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-14T14:52:33,862 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@6915083f{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-14T14:52:33,862 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-14T14:52:33,862 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@5cc2d6b{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-14T14:52:33,862 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@46b092e1{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c843459-ed19-f974-f761-d73dee21a91b/hadoop.log.dir/,STOPPED} 2024-11-14T14:52:33,863 WARN [BP-1055376130-172.17.0.2-1731595909704 heartbeating to localhost/127.0.0.1:34445 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-14T14:52:33,863 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-14T14:52:33,863 WARN [BP-1055376130-172.17.0.2-1731595909704 heartbeating to localhost/127.0.0.1:34445 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1055376130-172.17.0.2-1731595909704 (Datanode Uuid d55da410-599d-40d9-9c9f-2653bb81e8cf) service to localhost/127.0.0.1:34445 2024-11-14T14:52:33,863 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-14T14:52:33,864 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c843459-ed19-f974-f761-d73dee21a91b/cluster_fceed6e6-d3b5-652e-82f8-dddc2c9b7e2b/data/data3/current/BP-1055376130-172.17.0.2-1731595909704 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-14T14:52:33,864 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c843459-ed19-f974-f761-d73dee21a91b/cluster_fceed6e6-d3b5-652e-82f8-dddc2c9b7e2b/data/data4/current/BP-1055376130-172.17.0.2-1731595909704 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-14T14:52:33,865 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-14T14:52:33,866 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@330740de{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-14T14:52:33,867 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@7b24cab9{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-14T14:52:33,867 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-14T14:52:33,867 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4a359997{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-14T14:52:33,867 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@cf5a85e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c843459-ed19-f974-f761-d73dee21a91b/hadoop.log.dir/,STOPPED} 2024-11-14T14:52:33,868 WARN [BP-1055376130-172.17.0.2-1731595909704 heartbeating to localhost/127.0.0.1:34445 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-14T14:52:33,868 WARN [BP-1055376130-172.17.0.2-1731595909704 heartbeating to localhost/127.0.0.1:34445 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1055376130-172.17.0.2-1731595909704 (Datanode Uuid 63d3b466-1d56-4787-911e-c33e6d4192c1) service to localhost/127.0.0.1:34445 2024-11-14T14:52:33,869 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c843459-ed19-f974-f761-d73dee21a91b/cluster_fceed6e6-d3b5-652e-82f8-dddc2c9b7e2b/data/data2/current/BP-1055376130-172.17.0.2-1731595909704 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-14T14:52:33,869 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c843459-ed19-f974-f761-d73dee21a91b/cluster_fceed6e6-d3b5-652e-82f8-dddc2c9b7e2b/data/data1/current/BP-1055376130-172.17.0.2-1731595909704 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-14T14:52:33,869 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-14T14:52:33,869 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-14T14:52:33,869 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-14T14:52:33,875 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4c03e044{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-14T14:52:33,876 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@698721f1{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-14T14:52:33,876 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-14T14:52:33,876 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@652099c6{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-14T14:52:33,876 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@264e890{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c843459-ed19-f974-f761-d73dee21a91b/hadoop.log.dir/,STOPPED} 2024-11-14T14:52:33,885 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-14T14:52:33,931 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down