2024-12-03 23:24:38,541 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@20b2475a 2024-12-03 23:24:38,557 main DEBUG Took 0.013570 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-12-03 23:24:38,557 main DEBUG PluginManager 'Core' found 129 plugins 2024-12-03 23:24:38,558 main DEBUG PluginManager 'Level' found 0 plugins 2024-12-03 23:24:38,559 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-12-03 23:24:38,560 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 23:24:38,567 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-12-03 23:24:38,580 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 23:24:38,582 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 23:24:38,583 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 23:24:38,583 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 23:24:38,584 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 23:24:38,584 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 23:24:38,586 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 23:24:38,586 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 23:24:38,587 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 23:24:38,587 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 23:24:38,588 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 23:24:38,589 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 23:24:38,589 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 23:24:38,590 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 23:24:38,590 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 23:24:38,591 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 23:24:38,592 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 23:24:38,592 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 23:24:38,593 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 23:24:38,593 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 23:24:38,594 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 23:24:38,594 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 23:24:38,595 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 23:24:38,595 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 23:24:38,596 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 23:24:38,596 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-12-03 23:24:38,598 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 23:24:38,600 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-12-03 23:24:38,602 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-12-03 23:24:38,603 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-12-03 23:24:38,604 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-12-03 23:24:38,605 main DEBUG PluginManager 'Converter' found 47 plugins 2024-12-03 23:24:38,616 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-12-03 23:24:38,620 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-12-03 23:24:38,622 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-12-03 23:24:38,622 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-12-03 23:24:38,623 main DEBUG createAppenders(={Console}) 2024-12-03 23:24:38,624 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@20b2475a initialized 2024-12-03 23:24:38,624 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@20b2475a 2024-12-03 23:24:38,625 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@20b2475a OK. 2024-12-03 23:24:38,626 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-12-03 23:24:38,626 main DEBUG OutputStream closed 2024-12-03 23:24:38,626 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-12-03 23:24:38,627 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-12-03 23:24:38,627 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@4310d43 OK 2024-12-03 23:24:38,712 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-12-03 23:24:38,715 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-12-03 23:24:38,716 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-12-03 23:24:38,717 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-12-03 23:24:38,718 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-12-03 23:24:38,718 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-12-03 23:24:38,719 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-12-03 23:24:38,719 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-12-03 23:24:38,720 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-12-03 23:24:38,720 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-12-03 23:24:38,721 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-12-03 23:24:38,721 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-12-03 23:24:38,722 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-12-03 23:24:38,722 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-12-03 23:24:38,723 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-12-03 23:24:38,723 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-12-03 23:24:38,723 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-12-03 23:24:38,724 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-12-03 23:24:38,728 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-03 23:24:38,728 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@12f9af83) with optional ClassLoader: null 2024-12-03 23:24:38,728 main DEBUG Shutdown hook enabled. Registering a new one. 2024-12-03 23:24:38,729 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@12f9af83] started OK. 2024-12-03T23:24:38,963 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/246ab1eb-e73c-fcd7-a82a-bcf4c34f0add 2024-12-03 23:24:38,966 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-12-03 23:24:38,966 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-03T23:24:38,975 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay timeout: 13 mins 2024-12-03T23:24:38,981 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplayCompressed timeout: 13 mins 2024-12-03T23:24:39,000 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-12-03T23:24:39,047 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-12-03T23:24:39,047 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-12-03T23:24:39,062 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=3, rsPorts=, rsClass=null, numDataNodes=3, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-12-03T23:24:39,076 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/246ab1eb-e73c-fcd7-a82a-bcf4c34f0add/cluster_eb757c39-dfd9-3b39-78fd-eea66f4cff1d, deleteOnExit=true 2024-12-03T23:24:39,076 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-12-03T23:24:39,077 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/246ab1eb-e73c-fcd7-a82a-bcf4c34f0add/test.cache.data in system properties and HBase conf 2024-12-03T23:24:39,077 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/246ab1eb-e73c-fcd7-a82a-bcf4c34f0add/hadoop.tmp.dir in system properties and HBase conf 2024-12-03T23:24:39,078 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/246ab1eb-e73c-fcd7-a82a-bcf4c34f0add/hadoop.log.dir in system properties and HBase conf 2024-12-03T23:24:39,078 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/246ab1eb-e73c-fcd7-a82a-bcf4c34f0add/mapreduce.cluster.local.dir in system properties and HBase conf 2024-12-03T23:24:39,079 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/246ab1eb-e73c-fcd7-a82a-bcf4c34f0add/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-12-03T23:24:39,079 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-12-03T23:24:39,158 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-12-03T23:24:39,248 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-03T23:24:39,251 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/246ab1eb-e73c-fcd7-a82a-bcf4c34f0add/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-12-03T23:24:39,252 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/246ab1eb-e73c-fcd7-a82a-bcf4c34f0add/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-12-03T23:24:39,252 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/246ab1eb-e73c-fcd7-a82a-bcf4c34f0add/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-12-03T23:24:39,253 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/246ab1eb-e73c-fcd7-a82a-bcf4c34f0add/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-03T23:24:39,253 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/246ab1eb-e73c-fcd7-a82a-bcf4c34f0add/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-12-03T23:24:39,253 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/246ab1eb-e73c-fcd7-a82a-bcf4c34f0add/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-12-03T23:24:39,254 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/246ab1eb-e73c-fcd7-a82a-bcf4c34f0add/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-03T23:24:39,254 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/246ab1eb-e73c-fcd7-a82a-bcf4c34f0add/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-03T23:24:39,255 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/246ab1eb-e73c-fcd7-a82a-bcf4c34f0add/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-12-03T23:24:39,255 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/246ab1eb-e73c-fcd7-a82a-bcf4c34f0add/nfs.dump.dir in system properties and HBase conf 2024-12-03T23:24:39,255 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/246ab1eb-e73c-fcd7-a82a-bcf4c34f0add/java.io.tmpdir in system properties and HBase conf 2024-12-03T23:24:39,256 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/246ab1eb-e73c-fcd7-a82a-bcf4c34f0add/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-03T23:24:39,256 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/246ab1eb-e73c-fcd7-a82a-bcf4c34f0add/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-12-03T23:24:39,256 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/246ab1eb-e73c-fcd7-a82a-bcf4c34f0add/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-12-03T23:24:40,067 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-12-03T23:24:40,148 INFO [Time-limited test {}] log.Log(170): Logging initialized @2318ms to org.eclipse.jetty.util.log.Slf4jLog 2024-12-03T23:24:40,223 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-03T23:24:40,290 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-03T23:24:40,314 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-03T23:24:40,314 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-03T23:24:40,315 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-12-03T23:24:40,328 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-03T23:24:40,331 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6630be9c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/246ab1eb-e73c-fcd7-a82a-bcf4c34f0add/hadoop.log.dir/,AVAILABLE} 2024-12-03T23:24:40,332 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@70596a48{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-03T23:24:40,524 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@1843526c{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/246ab1eb-e73c-fcd7-a82a-bcf4c34f0add/java.io.tmpdir/jetty-localhost-38579-hadoop-hdfs-3_4_1-tests_jar-_-any-9876040052258234383/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-03T23:24:40,531 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@776c4d96{HTTP/1.1, (http/1.1)}{localhost:38579} 2024-12-03T23:24:40,531 INFO [Time-limited test {}] server.Server(415): Started @2701ms 2024-12-03T23:24:40,916 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-03T23:24:40,922 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-03T23:24:40,923 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-03T23:24:40,923 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-03T23:24:40,924 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-12-03T23:24:40,925 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4b895d95{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/246ab1eb-e73c-fcd7-a82a-bcf4c34f0add/hadoop.log.dir/,AVAILABLE} 2024-12-03T23:24:40,925 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3d33512e{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-03T23:24:41,045 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4dc44ec6{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/246ab1eb-e73c-fcd7-a82a-bcf4c34f0add/java.io.tmpdir/jetty-localhost-43725-hadoop-hdfs-3_4_1-tests_jar-_-any-12986651277725577136/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-03T23:24:41,046 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@17f6e478{HTTP/1.1, (http/1.1)}{localhost:43725} 2024-12-03T23:24:41,046 INFO [Time-limited test {}] server.Server(415): Started @3217ms 2024-12-03T23:24:41,101 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-03T23:24:41,232 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-03T23:24:41,239 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-03T23:24:41,240 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-03T23:24:41,240 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-03T23:24:41,240 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-12-03T23:24:41,243 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@e971547{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/246ab1eb-e73c-fcd7-a82a-bcf4c34f0add/hadoop.log.dir/,AVAILABLE} 2024-12-03T23:24:41,244 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3264f7d{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-03T23:24:41,372 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@28e1ba78{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/246ab1eb-e73c-fcd7-a82a-bcf4c34f0add/java.io.tmpdir/jetty-localhost-32993-hadoop-hdfs-3_4_1-tests_jar-_-any-10320175084092576644/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-03T23:24:41,373 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@702c0733{HTTP/1.1, (http/1.1)}{localhost:32993} 2024-12-03T23:24:41,374 INFO [Time-limited test {}] server.Server(415): Started @3544ms 2024-12-03T23:24:41,377 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-03T23:24:41,436 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-03T23:24:41,445 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-03T23:24:41,449 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-03T23:24:41,450 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-03T23:24:41,450 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-12-03T23:24:41,451 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1bf0fdb4{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/246ab1eb-e73c-fcd7-a82a-bcf4c34f0add/hadoop.log.dir/,AVAILABLE} 2024-12-03T23:24:41,452 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6853f5bc{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-03T23:24:41,577 WARN [Thread-107 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/246ab1eb-e73c-fcd7-a82a-bcf4c34f0add/cluster_eb757c39-dfd9-3b39-78fd-eea66f4cff1d/data/data2/current/BP-2057003906-172.17.0.2-1733268279817/current, will proceed with Du for space computation calculation, 2024-12-03T23:24:41,577 WARN [Thread-105 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/246ab1eb-e73c-fcd7-a82a-bcf4c34f0add/cluster_eb757c39-dfd9-3b39-78fd-eea66f4cff1d/data/data1/current/BP-2057003906-172.17.0.2-1733268279817/current, will proceed with Du for space computation calculation, 2024-12-03T23:24:41,577 WARN [Thread-108 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/246ab1eb-e73c-fcd7-a82a-bcf4c34f0add/cluster_eb757c39-dfd9-3b39-78fd-eea66f4cff1d/data/data4/current/BP-2057003906-172.17.0.2-1733268279817/current, will proceed with Du for space computation calculation, 2024-12-03T23:24:41,577 WARN [Thread-106 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/246ab1eb-e73c-fcd7-a82a-bcf4c34f0add/cluster_eb757c39-dfd9-3b39-78fd-eea66f4cff1d/data/data3/current/BP-2057003906-172.17.0.2-1733268279817/current, will proceed with Du for space computation calculation, 2024-12-03T23:24:41,587 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4290616c{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/246ab1eb-e73c-fcd7-a82a-bcf4c34f0add/java.io.tmpdir/jetty-localhost-35951-hadoop-hdfs-3_4_1-tests_jar-_-any-11543061614721231252/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-03T23:24:41,588 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@5e25d2b6{HTTP/1.1, (http/1.1)}{localhost:35951} 2024-12-03T23:24:41,588 INFO [Time-limited test {}] server.Server(415): Started @3758ms 2024-12-03T23:24:41,590 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-03T23:24:41,634 WARN [Thread-81 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-03T23:24:41,634 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-03T23:24:41,720 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xf72d9255ed0f2ef7 with lease ID 0xa8908c0f4fa6d3: Processing first storage report for DS-e6270720-ef81-458e-b5d6-959d16c4c378 from datanode DatanodeRegistration(127.0.0.1:39829, datanodeUuid=d14c7b45-d11a-4744-a24a-fef0e3de8324, infoPort=38065, infoSecurePort=0, ipcPort=41371, storageInfo=lv=-57;cid=testClusterID;nsid=703925958;c=1733268279817) 2024-12-03T23:24:41,721 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xf72d9255ed0f2ef7 with lease ID 0xa8908c0f4fa6d3: from storage DS-e6270720-ef81-458e-b5d6-959d16c4c378 node DatanodeRegistration(127.0.0.1:39829, datanodeUuid=d14c7b45-d11a-4744-a24a-fef0e3de8324, infoPort=38065, infoSecurePort=0, ipcPort=41371, storageInfo=lv=-57;cid=testClusterID;nsid=703925958;c=1733268279817), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-12-03T23:24:41,721 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x5beb63280411adc7 with lease ID 0xa8908c0f4fa6d2: Processing first storage report for DS-d55e5535-c0ce-4dc2-a45c-8e70eb010e56 from datanode DatanodeRegistration(127.0.0.1:45491, datanodeUuid=31ac79b2-eca3-47f4-bfac-d43617ab8328, infoPort=34451, infoSecurePort=0, ipcPort=40347, storageInfo=lv=-57;cid=testClusterID;nsid=703925958;c=1733268279817) 2024-12-03T23:24:41,722 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x5beb63280411adc7 with lease ID 0xa8908c0f4fa6d2: from storage DS-d55e5535-c0ce-4dc2-a45c-8e70eb010e56 node DatanodeRegistration(127.0.0.1:45491, datanodeUuid=31ac79b2-eca3-47f4-bfac-d43617ab8328, infoPort=34451, infoSecurePort=0, ipcPort=40347, storageInfo=lv=-57;cid=testClusterID;nsid=703925958;c=1733268279817), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-12-03T23:24:41,722 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xf72d9255ed0f2ef7 with lease ID 0xa8908c0f4fa6d3: Processing first storage report for DS-43485ca1-fd8e-465c-a6e0-abf19c2273c3 from datanode DatanodeRegistration(127.0.0.1:39829, datanodeUuid=d14c7b45-d11a-4744-a24a-fef0e3de8324, infoPort=38065, infoSecurePort=0, ipcPort=41371, storageInfo=lv=-57;cid=testClusterID;nsid=703925958;c=1733268279817) 2024-12-03T23:24:41,722 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xf72d9255ed0f2ef7 with lease ID 0xa8908c0f4fa6d3: from storage DS-43485ca1-fd8e-465c-a6e0-abf19c2273c3 node DatanodeRegistration(127.0.0.1:39829, datanodeUuid=d14c7b45-d11a-4744-a24a-fef0e3de8324, infoPort=38065, infoSecurePort=0, ipcPort=41371, storageInfo=lv=-57;cid=testClusterID;nsid=703925958;c=1733268279817), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-03T23:24:41,722 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x5beb63280411adc7 with lease ID 0xa8908c0f4fa6d2: Processing first storage report for DS-9b1f9cb0-8b31-4b19-8859-3db325eabf67 from datanode DatanodeRegistration(127.0.0.1:45491, datanodeUuid=31ac79b2-eca3-47f4-bfac-d43617ab8328, infoPort=34451, infoSecurePort=0, ipcPort=40347, storageInfo=lv=-57;cid=testClusterID;nsid=703925958;c=1733268279817) 2024-12-03T23:24:41,722 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x5beb63280411adc7 with lease ID 0xa8908c0f4fa6d2: from storage DS-9b1f9cb0-8b31-4b19-8859-3db325eabf67 node DatanodeRegistration(127.0.0.1:45491, datanodeUuid=31ac79b2-eca3-47f4-bfac-d43617ab8328, infoPort=34451, infoSecurePort=0, ipcPort=40347, storageInfo=lv=-57;cid=testClusterID;nsid=703925958;c=1733268279817), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-03T23:24:41,738 WARN [Thread-139 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/246ab1eb-e73c-fcd7-a82a-bcf4c34f0add/cluster_eb757c39-dfd9-3b39-78fd-eea66f4cff1d/data/data5/current/BP-2057003906-172.17.0.2-1733268279817/current, will proceed with Du for space computation calculation, 2024-12-03T23:24:41,739 WARN [Thread-140 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/246ab1eb-e73c-fcd7-a82a-bcf4c34f0add/cluster_eb757c39-dfd9-3b39-78fd-eea66f4cff1d/data/data6/current/BP-2057003906-172.17.0.2-1733268279817/current, will proceed with Du for space computation calculation, 2024-12-03T23:24:41,765 WARN [Thread-123 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-03T23:24:41,771 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x14dd28eaaabe97c with lease ID 0xa8908c0f4fa6d4: Processing first storage report for DS-459d168f-61f7-403b-9139-91eca369c54e from datanode DatanodeRegistration(127.0.0.1:41487, datanodeUuid=30e7d33a-aa1d-4010-b6b0-785ae5df5297, infoPort=36361, infoSecurePort=0, ipcPort=36487, storageInfo=lv=-57;cid=testClusterID;nsid=703925958;c=1733268279817) 2024-12-03T23:24:41,772 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x14dd28eaaabe97c with lease ID 0xa8908c0f4fa6d4: from storage DS-459d168f-61f7-403b-9139-91eca369c54e node DatanodeRegistration(127.0.0.1:41487, datanodeUuid=30e7d33a-aa1d-4010-b6b0-785ae5df5297, infoPort=36361, infoSecurePort=0, ipcPort=36487, storageInfo=lv=-57;cid=testClusterID;nsid=703925958;c=1733268279817), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-12-03T23:24:41,772 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x14dd28eaaabe97c with lease ID 0xa8908c0f4fa6d4: Processing first storage report for DS-7a6ca814-75bf-4237-9704-44642da249e5 from datanode DatanodeRegistration(127.0.0.1:41487, datanodeUuid=30e7d33a-aa1d-4010-b6b0-785ae5df5297, infoPort=36361, infoSecurePort=0, ipcPort=36487, storageInfo=lv=-57;cid=testClusterID;nsid=703925958;c=1733268279817) 2024-12-03T23:24:41,772 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x14dd28eaaabe97c with lease ID 0xa8908c0f4fa6d4: from storage DS-7a6ca814-75bf-4237-9704-44642da249e5 node DatanodeRegistration(127.0.0.1:41487, datanodeUuid=30e7d33a-aa1d-4010-b6b0-785ae5df5297, infoPort=36361, infoSecurePort=0, ipcPort=36487, storageInfo=lv=-57;cid=testClusterID;nsid=703925958;c=1733268279817), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-03T23:24:41,986 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/246ab1eb-e73c-fcd7-a82a-bcf4c34f0add 2024-12-03T23:24:42,083 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/246ab1eb-e73c-fcd7-a82a-bcf4c34f0add/cluster_eb757c39-dfd9-3b39-78fd-eea66f4cff1d/zookeeper_0, clientPort=51247, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/246ab1eb-e73c-fcd7-a82a-bcf4c34f0add/cluster_eb757c39-dfd9-3b39-78fd-eea66f4cff1d/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/246ab1eb-e73c-fcd7-a82a-bcf4c34f0add/cluster_eb757c39-dfd9-3b39-78fd-eea66f4cff1d/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-12-03T23:24:42,093 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=51247 2024-12-03T23:24:42,109 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-03T23:24:42,112 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-03T23:24:42,325 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45491 is added to blk_1073741825_1001 (size=7) 2024-12-03T23:24:42,326 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41487 is added to blk_1073741825_1001 (size=7) 2024-12-03T23:24:42,327 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39829 is added to blk_1073741825_1001 (size=7) 2024-12-03T23:24:42,732 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:35563/user/jenkins/test-data/e9061e95-7d9e-c12a-b652-9e8a1d87f0cf with version=8 2024-12-03T23:24:42,732 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:35563/user/jenkins/test-data/e9061e95-7d9e-c12a-b652-9e8a1d87f0cf/hbase-staging 2024-12-03T23:24:43,039 INFO [Time-limited test {}] client.ConnectionUtils(128): master/5f9d64419733:0 server-side Connection retries=45 2024-12-03T23:24:43,050 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-03T23:24:43,050 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-03T23:24:43,055 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-03T23:24:43,055 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-03T23:24:43,056 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-03T23:24:43,197 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-12-03T23:24:43,262 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-12-03T23:24:43,272 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-12-03T23:24:43,276 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-03T23:24:43,305 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 12577 (auto-detected) 2024-12-03T23:24:43,306 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-12-03T23:24:43,327 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:41995 2024-12-03T23:24:43,350 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:41995 connecting to ZooKeeper ensemble=127.0.0.1:51247 2024-12-03T23:24:43,382 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:419950x0, quorum=127.0.0.1:51247, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-03T23:24:43,384 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:41995-0x1017aa5055b0000 connected 2024-12-03T23:24:43,413 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-03T23:24:43,416 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-03T23:24:43,429 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:41995-0x1017aa5055b0000, quorum=127.0.0.1:51247, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-03T23:24:43,433 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:35563/user/jenkins/test-data/e9061e95-7d9e-c12a-b652-9e8a1d87f0cf, hbase.cluster.distributed=false 2024-12-03T23:24:43,457 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:41995-0x1017aa5055b0000, quorum=127.0.0.1:51247, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-03T23:24:43,462 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=41995 2024-12-03T23:24:43,462 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=41995 2024-12-03T23:24:43,463 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=41995 2024-12-03T23:24:43,464 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=41995 2024-12-03T23:24:43,464 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=41995 2024-12-03T23:24:43,576 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/5f9d64419733:0 server-side Connection retries=45 2024-12-03T23:24:43,578 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-03T23:24:43,579 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-03T23:24:43,579 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-03T23:24:43,579 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-03T23:24:43,579 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-03T23:24:43,582 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-03T23:24:43,584 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-03T23:24:43,585 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:40969 2024-12-03T23:24:43,587 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:40969 connecting to ZooKeeper ensemble=127.0.0.1:51247 2024-12-03T23:24:43,588 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-03T23:24:43,592 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-03T23:24:43,599 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:409690x0, quorum=127.0.0.1:51247, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-03T23:24:43,600 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:40969-0x1017aa5055b0001 connected 2024-12-03T23:24:43,601 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40969-0x1017aa5055b0001, quorum=127.0.0.1:51247, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-03T23:24:43,605 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-03T23:24:43,613 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-03T23:24:43,615 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40969-0x1017aa5055b0001, quorum=127.0.0.1:51247, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-03T23:24:43,622 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40969-0x1017aa5055b0001, quorum=127.0.0.1:51247, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-03T23:24:43,622 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=40969 2024-12-03T23:24:43,623 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=40969 2024-12-03T23:24:43,623 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=40969 2024-12-03T23:24:43,624 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=40969 2024-12-03T23:24:43,624 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=40969 2024-12-03T23:24:43,642 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/5f9d64419733:0 server-side Connection retries=45 2024-12-03T23:24:43,642 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-03T23:24:43,643 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-03T23:24:43,643 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-03T23:24:43,643 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-03T23:24:43,643 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-03T23:24:43,644 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-03T23:24:43,644 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-03T23:24:43,645 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:44215 2024-12-03T23:24:43,647 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:44215 connecting to ZooKeeper ensemble=127.0.0.1:51247 2024-12-03T23:24:43,648 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-03T23:24:43,651 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-03T23:24:43,657 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:442150x0, quorum=127.0.0.1:51247, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-03T23:24:43,657 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:442150x0, quorum=127.0.0.1:51247, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-03T23:24:43,658 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:44215-0x1017aa5055b0002 connected 2024-12-03T23:24:43,658 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-03T23:24:43,659 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-03T23:24:43,660 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:44215-0x1017aa5055b0002, quorum=127.0.0.1:51247, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-03T23:24:43,662 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:44215-0x1017aa5055b0002, quorum=127.0.0.1:51247, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-03T23:24:43,663 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=44215 2024-12-03T23:24:43,663 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=44215 2024-12-03T23:24:43,663 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=44215 2024-12-03T23:24:43,664 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=44215 2024-12-03T23:24:43,664 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=44215 2024-12-03T23:24:43,681 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/5f9d64419733:0 server-side Connection retries=45 2024-12-03T23:24:43,681 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-03T23:24:43,681 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-03T23:24:43,681 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-03T23:24:43,681 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-03T23:24:43,682 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-03T23:24:43,682 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-03T23:24:43,682 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-03T23:24:43,683 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:44411 2024-12-03T23:24:43,684 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:44411 connecting to ZooKeeper ensemble=127.0.0.1:51247 2024-12-03T23:24:43,685 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-03T23:24:43,687 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-03T23:24:43,693 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:444110x0, quorum=127.0.0.1:51247, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-03T23:24:43,694 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:44411-0x1017aa5055b0003 connected 2024-12-03T23:24:43,694 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:44411-0x1017aa5055b0003, quorum=127.0.0.1:51247, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-03T23:24:43,694 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-03T23:24:43,696 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-03T23:24:43,697 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:44411-0x1017aa5055b0003, quorum=127.0.0.1:51247, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-03T23:24:43,699 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:44411-0x1017aa5055b0003, quorum=127.0.0.1:51247, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-03T23:24:43,700 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=44411 2024-12-03T23:24:43,701 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=44411 2024-12-03T23:24:43,701 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=44411 2024-12-03T23:24:43,702 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=44411 2024-12-03T23:24:43,702 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=44411 2024-12-03T23:24:43,718 DEBUG [M:0;5f9d64419733:41995 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;5f9d64419733:41995 2024-12-03T23:24:43,719 INFO [master/5f9d64419733:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/5f9d64419733,41995,1733268282840 2024-12-03T23:24:43,726 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41995-0x1017aa5055b0000, quorum=127.0.0.1:51247, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-03T23:24:43,726 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40969-0x1017aa5055b0001, quorum=127.0.0.1:51247, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-03T23:24:43,726 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44411-0x1017aa5055b0003, quorum=127.0.0.1:51247, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-03T23:24:43,726 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44215-0x1017aa5055b0002, quorum=127.0.0.1:51247, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-03T23:24:43,729 DEBUG [master/5f9d64419733:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:41995-0x1017aa5055b0000, quorum=127.0.0.1:51247, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/5f9d64419733,41995,1733268282840 2024-12-03T23:24:43,755 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44215-0x1017aa5055b0002, quorum=127.0.0.1:51247, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-03T23:24:43,755 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40969-0x1017aa5055b0001, quorum=127.0.0.1:51247, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-03T23:24:43,755 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41995-0x1017aa5055b0000, quorum=127.0.0.1:51247, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T23:24:43,755 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44411-0x1017aa5055b0003, quorum=127.0.0.1:51247, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-03T23:24:43,755 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40969-0x1017aa5055b0001, quorum=127.0.0.1:51247, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T23:24:43,755 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44215-0x1017aa5055b0002, quorum=127.0.0.1:51247, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T23:24:43,755 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44411-0x1017aa5055b0003, quorum=127.0.0.1:51247, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T23:24:43,756 DEBUG [master/5f9d64419733:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:41995-0x1017aa5055b0000, quorum=127.0.0.1:51247, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-03T23:24:43,757 INFO [master/5f9d64419733:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/5f9d64419733,41995,1733268282840 from backup master directory 2024-12-03T23:24:43,760 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41995-0x1017aa5055b0000, quorum=127.0.0.1:51247, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/5f9d64419733,41995,1733268282840 2024-12-03T23:24:43,760 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40969-0x1017aa5055b0001, quorum=127.0.0.1:51247, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-03T23:24:43,760 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44215-0x1017aa5055b0002, quorum=127.0.0.1:51247, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-03T23:24:43,760 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41995-0x1017aa5055b0000, quorum=127.0.0.1:51247, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-03T23:24:43,760 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44411-0x1017aa5055b0003, quorum=127.0.0.1:51247, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-03T23:24:43,761 WARN [master/5f9d64419733:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-03T23:24:43,762 INFO [master/5f9d64419733:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=5f9d64419733,41995,1733268282840 2024-12-03T23:24:43,764 INFO [master/5f9d64419733:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-12-03T23:24:43,766 INFO [master/5f9d64419733:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-12-03T23:24:43,825 DEBUG [master/5f9d64419733:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:35563/user/jenkins/test-data/e9061e95-7d9e-c12a-b652-9e8a1d87f0cf/hbase.id] with ID: c861c93d-86c4-4933-988c-7abbfe26c2c0 2024-12-03T23:24:43,825 DEBUG [master/5f9d64419733:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:35563/user/jenkins/test-data/e9061e95-7d9e-c12a-b652-9e8a1d87f0cf/.tmp/hbase.id 2024-12-03T23:24:43,839 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45491 is added to blk_1073741826_1002 (size=42) 2024-12-03T23:24:43,839 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41487 is added to blk_1073741826_1002 (size=42) 2024-12-03T23:24:43,840 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39829 is added to blk_1073741826_1002 (size=42) 2024-12-03T23:24:43,841 DEBUG [master/5f9d64419733:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:35563/user/jenkins/test-data/e9061e95-7d9e-c12a-b652-9e8a1d87f0cf/.tmp/hbase.id]:[hdfs://localhost:35563/user/jenkins/test-data/e9061e95-7d9e-c12a-b652-9e8a1d87f0cf/hbase.id] 2024-12-03T23:24:43,888 INFO [master/5f9d64419733:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-03T23:24:43,893 INFO [master/5f9d64419733:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-12-03T23:24:43,911 INFO [master/5f9d64419733:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 16ms. 2024-12-03T23:24:43,916 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41995-0x1017aa5055b0000, quorum=127.0.0.1:51247, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T23:24:43,916 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40969-0x1017aa5055b0001, quorum=127.0.0.1:51247, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T23:24:43,916 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44215-0x1017aa5055b0002, quorum=127.0.0.1:51247, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T23:24:43,916 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44411-0x1017aa5055b0003, quorum=127.0.0.1:51247, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T23:24:43,935 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41487 is added to blk_1073741827_1003 (size=196) 2024-12-03T23:24:43,936 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39829 is added to blk_1073741827_1003 (size=196) 2024-12-03T23:24:43,936 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45491 is added to blk_1073741827_1003 (size=196) 2024-12-03T23:24:43,954 INFO [master/5f9d64419733:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-03T23:24:43,956 INFO [master/5f9d64419733:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-12-03T23:24:43,962 INFO [master/5f9d64419733:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-03T23:24:43,996 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41487 is added to blk_1073741828_1004 (size=1189) 2024-12-03T23:24:43,996 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45491 is added to blk_1073741828_1004 (size=1189) 2024-12-03T23:24:43,996 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39829 is added to blk_1073741828_1004 (size=1189) 2024-12-03T23:24:44,015 INFO [master/5f9d64419733:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:35563/user/jenkins/test-data/e9061e95-7d9e-c12a-b652-9e8a1d87f0cf/MasterData/data/master/store 2024-12-03T23:24:44,037 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45491 is added to blk_1073741829_1005 (size=34) 2024-12-03T23:24:44,037 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39829 is added to blk_1073741829_1005 (size=34) 2024-12-03T23:24:44,038 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41487 is added to blk_1073741829_1005 (size=34) 2024-12-03T23:24:44,043 INFO [master/5f9d64419733:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-12-03T23:24:44,047 DEBUG [master/5f9d64419733:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T23:24:44,048 DEBUG [master/5f9d64419733:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-03T23:24:44,049 INFO [master/5f9d64419733:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-03T23:24:44,049 DEBUG [master/5f9d64419733:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-03T23:24:44,051 DEBUG [master/5f9d64419733:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-03T23:24:44,051 DEBUG [master/5f9d64419733:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-03T23:24:44,052 INFO [master/5f9d64419733:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-03T23:24:44,053 DEBUG [master/5f9d64419733:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733268284048Disabling compacts and flushes for region at 1733268284048Disabling writes for close at 1733268284051 (+3 ms)Writing region close event to WAL at 1733268284051Closed at 1733268284051 2024-12-03T23:24:44,055 WARN [master/5f9d64419733:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:35563/user/jenkins/test-data/e9061e95-7d9e-c12a-b652-9e8a1d87f0cf/MasterData/data/master/store/.initializing 2024-12-03T23:24:44,056 DEBUG [master/5f9d64419733:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:35563/user/jenkins/test-data/e9061e95-7d9e-c12a-b652-9e8a1d87f0cf/MasterData/WALs/5f9d64419733,41995,1733268282840 2024-12-03T23:24:44,066 INFO [master/5f9d64419733:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-03T23:24:44,084 INFO [master/5f9d64419733:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=5f9d64419733%2C41995%2C1733268282840, suffix=, logDir=hdfs://localhost:35563/user/jenkins/test-data/e9061e95-7d9e-c12a-b652-9e8a1d87f0cf/MasterData/WALs/5f9d64419733,41995,1733268282840, archiveDir=hdfs://localhost:35563/user/jenkins/test-data/e9061e95-7d9e-c12a-b652-9e8a1d87f0cf/MasterData/oldWALs, maxLogs=10 2024-12-03T23:24:44,125 DEBUG [master/5f9d64419733:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/e9061e95-7d9e-c12a-b652-9e8a1d87f0cf/MasterData/WALs/5f9d64419733,41995,1733268282840/5f9d64419733%2C41995%2C1733268282840.1733268284090, exclude list is [], retry=0 2024-12-03T23:24:44,146 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.initialize(FanOutOneBlockAsyncDFSOutputHelper.java:413) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$5.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:472) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$5.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:467) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.NettyFutureUtils.lambda$addListener$0(NettyFutureUtils.java:56) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setSuccess0(DefaultPromise.java:625) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.trySuccess(DefaultPromise.java:105) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPromise.trySuccess(DefaultChannelPromise.java:84) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:658) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:696) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T23:24:44,151 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45491,DS-d55e5535-c0ce-4dc2-a45c-8e70eb010e56,DISK] 2024-12-03T23:24:44,151 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39829,DS-e6270720-ef81-458e-b5d6-959d16c4c378,DISK] 2024-12-03T23:24:44,151 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41487,DS-459d168f-61f7-403b-9139-91eca369c54e,DISK] 2024-12-03T23:24:44,154 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-12-03T23:24:44,195 INFO [master/5f9d64419733:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/e9061e95-7d9e-c12a-b652-9e8a1d87f0cf/MasterData/WALs/5f9d64419733,41995,1733268282840/5f9d64419733%2C41995%2C1733268282840.1733268284090 2024-12-03T23:24:44,196 DEBUG [master/5f9d64419733:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:38065:38065),(127.0.0.1/127.0.0.1:34451:34451),(127.0.0.1/127.0.0.1:36361:36361)] 2024-12-03T23:24:44,197 DEBUG [master/5f9d64419733:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-12-03T23:24:44,197 DEBUG [master/5f9d64419733:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T23:24:44,200 DEBUG [master/5f9d64419733:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-12-03T23:24:44,202 DEBUG [master/5f9d64419733:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-12-03T23:24:44,239 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-12-03T23:24:44,265 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-12-03T23:24:44,269 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T23:24:44,272 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-03T23:24:44,272 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-12-03T23:24:44,275 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-12-03T23:24:44,276 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T23:24:44,277 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T23:24:44,277 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-12-03T23:24:44,280 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-12-03T23:24:44,280 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T23:24:44,281 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T23:24:44,281 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-12-03T23:24:44,284 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-12-03T23:24:44,285 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T23:24:44,286 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T23:24:44,286 DEBUG [master/5f9d64419733:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-12-03T23:24:44,289 DEBUG [master/5f9d64419733:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35563/user/jenkins/test-data/e9061e95-7d9e-c12a-b652-9e8a1d87f0cf/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-03T23:24:44,290 DEBUG [master/5f9d64419733:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35563/user/jenkins/test-data/e9061e95-7d9e-c12a-b652-9e8a1d87f0cf/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-03T23:24:44,295 DEBUG [master/5f9d64419733:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-12-03T23:24:44,295 DEBUG [master/5f9d64419733:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-12-03T23:24:44,298 DEBUG [master/5f9d64419733:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-03T23:24:44,301 DEBUG [master/5f9d64419733:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-12-03T23:24:44,305 DEBUG [master/5f9d64419733:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35563/user/jenkins/test-data/e9061e95-7d9e-c12a-b652-9e8a1d87f0cf/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T23:24:44,306 INFO [master/5f9d64419733:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66165187, jitterRate=-0.014061883091926575}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-03T23:24:44,312 DEBUG [master/5f9d64419733:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1733268284214Initializing all the Stores at 1733268284216 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733268284217 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733268284217Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733268284218 (+1 ms)Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733268284218Cleaning up temporary data from old regions at 1733268284295 (+77 ms)Region opened successfully at 1733268284312 (+17 ms) 2024-12-03T23:24:44,313 INFO [master/5f9d64419733:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-12-03T23:24:44,349 DEBUG [master/5f9d64419733:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@79f22941, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=5f9d64419733/172.17.0.2:0 2024-12-03T23:24:44,382 INFO [master/5f9d64419733:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-12-03T23:24:44,393 INFO [master/5f9d64419733:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-12-03T23:24:44,393 INFO [master/5f9d64419733:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-12-03T23:24:44,396 INFO [master/5f9d64419733:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-03T23:24:44,397 INFO [master/5f9d64419733:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-12-03T23:24:44,402 INFO [master/5f9d64419733:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-12-03T23:24:44,402 INFO [master/5f9d64419733:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-12-03T23:24:44,428 INFO [master/5f9d64419733:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-12-03T23:24:44,437 DEBUG [master/5f9d64419733:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41995-0x1017aa5055b0000, quorum=127.0.0.1:51247, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-12-03T23:24:44,439 DEBUG [master/5f9d64419733:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-12-03T23:24:44,442 INFO [master/5f9d64419733:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-12-03T23:24:44,443 DEBUG [master/5f9d64419733:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41995-0x1017aa5055b0000, quorum=127.0.0.1:51247, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-12-03T23:24:44,446 DEBUG [master/5f9d64419733:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-12-03T23:24:44,448 INFO [master/5f9d64419733:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-12-03T23:24:44,452 DEBUG [master/5f9d64419733:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41995-0x1017aa5055b0000, quorum=127.0.0.1:51247, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-12-03T23:24:44,453 DEBUG [master/5f9d64419733:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-12-03T23:24:44,454 DEBUG [master/5f9d64419733:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41995-0x1017aa5055b0000, quorum=127.0.0.1:51247, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-12-03T23:24:44,457 DEBUG [master/5f9d64419733:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-12-03T23:24:44,475 DEBUG [master/5f9d64419733:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41995-0x1017aa5055b0000, quorum=127.0.0.1:51247, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-12-03T23:24:44,477 DEBUG [master/5f9d64419733:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-12-03T23:24:44,481 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41995-0x1017aa5055b0000, quorum=127.0.0.1:51247, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-03T23:24:44,481 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44411-0x1017aa5055b0003, quorum=127.0.0.1:51247, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-03T23:24:44,481 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40969-0x1017aa5055b0001, quorum=127.0.0.1:51247, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-03T23:24:44,481 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44215-0x1017aa5055b0002, quorum=127.0.0.1:51247, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-03T23:24:44,481 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44411-0x1017aa5055b0003, quorum=127.0.0.1:51247, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T23:24:44,481 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41995-0x1017aa5055b0000, quorum=127.0.0.1:51247, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T23:24:44,481 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40969-0x1017aa5055b0001, quorum=127.0.0.1:51247, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T23:24:44,481 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44215-0x1017aa5055b0002, quorum=127.0.0.1:51247, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T23:24:44,484 INFO [master/5f9d64419733:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=5f9d64419733,41995,1733268282840, sessionid=0x1017aa5055b0000, setting cluster-up flag (Was=false) 2024-12-03T23:24:44,497 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44411-0x1017aa5055b0003, quorum=127.0.0.1:51247, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T23:24:44,497 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44215-0x1017aa5055b0002, quorum=127.0.0.1:51247, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T23:24:44,497 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40969-0x1017aa5055b0001, quorum=127.0.0.1:51247, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T23:24:44,497 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41995-0x1017aa5055b0000, quorum=127.0.0.1:51247, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T23:24:44,503 DEBUG [master/5f9d64419733:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-12-03T23:24:44,505 DEBUG [master/5f9d64419733:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=5f9d64419733,41995,1733268282840 2024-12-03T23:24:44,512 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41995-0x1017aa5055b0000, quorum=127.0.0.1:51247, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T23:24:44,512 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44411-0x1017aa5055b0003, quorum=127.0.0.1:51247, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T23:24:44,512 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44215-0x1017aa5055b0002, quorum=127.0.0.1:51247, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T23:24:44,512 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40969-0x1017aa5055b0001, quorum=127.0.0.1:51247, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T23:24:44,519 DEBUG [master/5f9d64419733:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-12-03T23:24:44,521 DEBUG [master/5f9d64419733:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=5f9d64419733,41995,1733268282840 2024-12-03T23:24:44,527 INFO [master/5f9d64419733:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:35563/user/jenkins/test-data/e9061e95-7d9e-c12a-b652-9e8a1d87f0cf/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-12-03T23:24:44,601 DEBUG [master/5f9d64419733:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-03T23:24:44,606 INFO [RS:0;5f9d64419733:40969 {}] regionserver.HRegionServer(746): ClusterId : c861c93d-86c4-4933-988c-7abbfe26c2c0 2024-12-03T23:24:44,606 INFO [RS:2;5f9d64419733:44411 {}] regionserver.HRegionServer(746): ClusterId : c861c93d-86c4-4933-988c-7abbfe26c2c0 2024-12-03T23:24:44,606 INFO [RS:1;5f9d64419733:44215 {}] regionserver.HRegionServer(746): ClusterId : c861c93d-86c4-4933-988c-7abbfe26c2c0 2024-12-03T23:24:44,609 DEBUG [RS:1;5f9d64419733:44215 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-03T23:24:44,609 DEBUG [RS:2;5f9d64419733:44411 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-03T23:24:44,609 DEBUG [RS:0;5f9d64419733:40969 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-03T23:24:44,611 INFO [master/5f9d64419733:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-12-03T23:24:44,615 DEBUG [RS:0;5f9d64419733:40969 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-03T23:24:44,615 DEBUG [RS:2;5f9d64419733:44411 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-03T23:24:44,615 DEBUG [RS:1;5f9d64419733:44215 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-03T23:24:44,615 DEBUG [RS:0;5f9d64419733:40969 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-03T23:24:44,615 DEBUG [RS:1;5f9d64419733:44215 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-03T23:24:44,615 DEBUG [RS:2;5f9d64419733:44411 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-03T23:24:44,619 DEBUG [RS:1;5f9d64419733:44215 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-03T23:24:44,619 DEBUG [RS:0;5f9d64419733:40969 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-03T23:24:44,619 DEBUG [RS:2;5f9d64419733:44411 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-03T23:24:44,619 INFO [master/5f9d64419733:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-12-03T23:24:44,620 DEBUG [RS:1;5f9d64419733:44215 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@ada8fef, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=5f9d64419733/172.17.0.2:0 2024-12-03T23:24:44,620 DEBUG [RS:0;5f9d64419733:40969 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@a7f6d85, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=5f9d64419733/172.17.0.2:0 2024-12-03T23:24:44,620 DEBUG [RS:2;5f9d64419733:44411 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@68252cca, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=5f9d64419733/172.17.0.2:0 2024-12-03T23:24:44,627 DEBUG [master/5f9d64419733:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 5f9d64419733,41995,1733268282840 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-12-03T23:24:44,636 DEBUG [master/5f9d64419733:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/5f9d64419733:0, corePoolSize=5, maxPoolSize=5 2024-12-03T23:24:44,637 DEBUG [master/5f9d64419733:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/5f9d64419733:0, corePoolSize=5, maxPoolSize=5 2024-12-03T23:24:44,637 DEBUG [master/5f9d64419733:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/5f9d64419733:0, corePoolSize=5, maxPoolSize=5 2024-12-03T23:24:44,637 DEBUG [master/5f9d64419733:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/5f9d64419733:0, corePoolSize=5, maxPoolSize=5 2024-12-03T23:24:44,637 DEBUG [master/5f9d64419733:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/5f9d64419733:0, corePoolSize=10, maxPoolSize=10 2024-12-03T23:24:44,637 DEBUG [master/5f9d64419733:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/5f9d64419733:0, corePoolSize=1, maxPoolSize=1 2024-12-03T23:24:44,638 DEBUG [master/5f9d64419733:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/5f9d64419733:0, corePoolSize=2, maxPoolSize=2 2024-12-03T23:24:44,638 DEBUG [master/5f9d64419733:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/5f9d64419733:0, corePoolSize=1, maxPoolSize=1 2024-12-03T23:24:44,639 DEBUG [RS:2;5f9d64419733:44411 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:2;5f9d64419733:44411 2024-12-03T23:24:44,639 DEBUG [RS:0;5f9d64419733:40969 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;5f9d64419733:40969 2024-12-03T23:24:44,641 DEBUG [RS:1;5f9d64419733:44215 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;5f9d64419733:44215 2024-12-03T23:24:44,642 INFO [RS:2;5f9d64419733:44411 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-03T23:24:44,642 INFO [RS:0;5f9d64419733:40969 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-03T23:24:44,642 INFO [RS:1;5f9d64419733:44215 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-03T23:24:44,642 INFO [RS:2;5f9d64419733:44411 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-03T23:24:44,642 INFO [RS:1;5f9d64419733:44215 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-03T23:24:44,642 INFO [RS:0;5f9d64419733:40969 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-03T23:24:44,643 DEBUG [RS:2;5f9d64419733:44411 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-03T23:24:44,643 DEBUG [RS:0;5f9d64419733:40969 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-03T23:24:44,643 DEBUG [RS:1;5f9d64419733:44215 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-03T23:24:44,644 INFO [master/5f9d64419733:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733268314644 2024-12-03T23:24:44,646 INFO [RS:0;5f9d64419733:40969 {}] regionserver.HRegionServer(2659): reportForDuty to master=5f9d64419733,41995,1733268282840 with port=40969, startcode=1733268283535 2024-12-03T23:24:44,646 INFO [RS:1;5f9d64419733:44215 {}] regionserver.HRegionServer(2659): reportForDuty to master=5f9d64419733,41995,1733268282840 with port=44215, startcode=1733268283642 2024-12-03T23:24:44,646 INFO [RS:2;5f9d64419733:44411 {}] regionserver.HRegionServer(2659): reportForDuty to master=5f9d64419733,41995,1733268282840 with port=44411, startcode=1733268283680 2024-12-03T23:24:44,646 INFO [master/5f9d64419733:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-12-03T23:24:44,647 INFO [master/5f9d64419733:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-12-03T23:24:44,651 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-03T23:24:44,652 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-12-03T23:24:44,652 INFO [master/5f9d64419733:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-12-03T23:24:44,652 INFO [master/5f9d64419733:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-12-03T23:24:44,653 INFO [master/5f9d64419733:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-12-03T23:24:44,653 INFO [master/5f9d64419733:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-12-03T23:24:44,654 INFO [master/5f9d64419733:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-03T23:24:44,657 INFO [master/5f9d64419733:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-12-03T23:24:44,658 INFO [master/5f9d64419733:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-12-03T23:24:44,658 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T23:24:44,658 INFO [master/5f9d64419733:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-12-03T23:24:44,658 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-12-03T23:24:44,661 DEBUG [RS:0;5f9d64419733:40969 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-03T23:24:44,661 DEBUG [RS:1;5f9d64419733:44215 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-03T23:24:44,661 DEBUG [RS:2;5f9d64419733:44411 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-03T23:24:44,662 INFO [master/5f9d64419733:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-12-03T23:24:44,662 INFO [master/5f9d64419733:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-12-03T23:24:44,667 DEBUG [master/5f9d64419733:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/5f9d64419733:0:becomeActiveMaster-HFileCleaner.large.0-1733268284664,5,FailOnTimeoutGroup] 2024-12-03T23:24:44,668 DEBUG [master/5f9d64419733:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/5f9d64419733:0:becomeActiveMaster-HFileCleaner.small.0-1733268284667,5,FailOnTimeoutGroup] 2024-12-03T23:24:44,668 INFO [master/5f9d64419733:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-03T23:24:44,668 INFO [master/5f9d64419733:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-12-03T23:24:44,669 INFO [master/5f9d64419733:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-12-03T23:24:44,670 INFO [master/5f9d64419733:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-12-03T23:24:44,703 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41487 is added to blk_1073741831_1007 (size=1321) 2024-12-03T23:24:44,703 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45491 is added to blk_1073741831_1007 (size=1321) 2024-12-03T23:24:44,704 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39829 is added to blk_1073741831_1007 (size=1321) 2024-12-03T23:24:44,708 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:35563/user/jenkins/test-data/e9061e95-7d9e-c12a-b652-9e8a1d87f0cf/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-12-03T23:24:44,709 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:35563/user/jenkins/test-data/e9061e95-7d9e-c12a-b652-9e8a1d87f0cf 2024-12-03T23:24:44,722 INFO [HMaster-EventLoopGroup-2-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57537, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.1 (auth:SIMPLE), service=RegionServerStatusService 2024-12-03T23:24:44,722 INFO [HMaster-EventLoopGroup-2-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38003, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.2 (auth:SIMPLE), service=RegionServerStatusService 2024-12-03T23:24:44,722 INFO [HMaster-EventLoopGroup-2-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45099, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-12-03T23:24:44,726 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39829 is added to blk_1073741832_1008 (size=32) 2024-12-03T23:24:44,727 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45491 is added to blk_1073741832_1008 (size=32) 2024-12-03T23:24:44,727 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41487 is added to blk_1073741832_1008 (size=32) 2024-12-03T23:24:44,729 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41995 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 5f9d64419733,40969,1733268283535 2024-12-03T23:24:44,729 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T23:24:44,731 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41995 {}] master.ServerManager(517): Registering regionserver=5f9d64419733,40969,1733268283535 2024-12-03T23:24:44,733 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-03T23:24:44,736 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-03T23:24:44,736 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T23:24:44,737 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-03T23:24:44,737 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-03T23:24:44,740 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-03T23:24:44,740 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T23:24:44,741 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-03T23:24:44,742 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-03T23:24:44,744 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-03T23:24:44,744 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41995 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 5f9d64419733,44411,1733268283680 2024-12-03T23:24:44,744 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T23:24:44,744 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41995 {}] master.ServerManager(517): Registering regionserver=5f9d64419733,44411,1733268283680 2024-12-03T23:24:44,745 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-03T23:24:44,746 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-03T23:24:44,749 DEBUG [RS:0;5f9d64419733:40969 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:35563/user/jenkins/test-data/e9061e95-7d9e-c12a-b652-9e8a1d87f0cf 2024-12-03T23:24:44,749 DEBUG [RS:0;5f9d64419733:40969 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:35563 2024-12-03T23:24:44,749 DEBUG [RS:0;5f9d64419733:40969 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-03T23:24:44,750 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-03T23:24:44,750 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41995 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 5f9d64419733,44215,1733268283642 2024-12-03T23:24:44,750 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T23:24:44,750 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41995 {}] master.ServerManager(517): Registering regionserver=5f9d64419733,44215,1733268283642 2024-12-03T23:24:44,751 DEBUG [RS:2;5f9d64419733:44411 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:35563/user/jenkins/test-data/e9061e95-7d9e-c12a-b652-9e8a1d87f0cf 2024-12-03T23:24:44,751 DEBUG [RS:2;5f9d64419733:44411 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:35563 2024-12-03T23:24:44,751 DEBUG [RS:2;5f9d64419733:44411 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-03T23:24:44,751 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-03T23:24:44,752 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-03T23:24:44,754 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35563/user/jenkins/test-data/e9061e95-7d9e-c12a-b652-9e8a1d87f0cf/data/hbase/meta/1588230740 2024-12-03T23:24:44,754 DEBUG [RS:1;5f9d64419733:44215 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:35563/user/jenkins/test-data/e9061e95-7d9e-c12a-b652-9e8a1d87f0cf 2024-12-03T23:24:44,754 DEBUG [RS:1;5f9d64419733:44215 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:35563 2024-12-03T23:24:44,754 DEBUG [RS:1;5f9d64419733:44215 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-03T23:24:44,755 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35563/user/jenkins/test-data/e9061e95-7d9e-c12a-b652-9e8a1d87f0cf/data/hbase/meta/1588230740 2024-12-03T23:24:44,757 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-03T23:24:44,757 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-03T23:24:44,758 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41995-0x1017aa5055b0000, quorum=127.0.0.1:51247, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-03T23:24:44,759 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-03T23:24:44,760 DEBUG [RS:0;5f9d64419733:40969 {}] zookeeper.ZKUtil(111): regionserver:40969-0x1017aa5055b0001, quorum=127.0.0.1:51247, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/5f9d64419733,40969,1733268283535 2024-12-03T23:24:44,760 DEBUG [RS:2;5f9d64419733:44411 {}] zookeeper.ZKUtil(111): regionserver:44411-0x1017aa5055b0003, quorum=127.0.0.1:51247, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/5f9d64419733,44411,1733268283680 2024-12-03T23:24:44,760 WARN [RS:0;5f9d64419733:40969 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-03T23:24:44,760 WARN [RS:2;5f9d64419733:44411 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-03T23:24:44,760 INFO [RS:0;5f9d64419733:40969 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-03T23:24:44,760 INFO [RS:2;5f9d64419733:44411 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-03T23:24:44,761 DEBUG [RS:1;5f9d64419733:44215 {}] zookeeper.ZKUtil(111): regionserver:44215-0x1017aa5055b0002, quorum=127.0.0.1:51247, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/5f9d64419733,44215,1733268283642 2024-12-03T23:24:44,761 WARN [RS:1;5f9d64419733:44215 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-03T23:24:44,761 INFO [RS:1;5f9d64419733:44215 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-03T23:24:44,761 DEBUG [RS:0;5f9d64419733:40969 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:35563/user/jenkins/test-data/e9061e95-7d9e-c12a-b652-9e8a1d87f0cf/WALs/5f9d64419733,40969,1733268283535 2024-12-03T23:24:44,761 DEBUG [RS:2;5f9d64419733:44411 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:35563/user/jenkins/test-data/e9061e95-7d9e-c12a-b652-9e8a1d87f0cf/WALs/5f9d64419733,44411,1733268283680 2024-12-03T23:24:44,761 DEBUG [RS:1;5f9d64419733:44215 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:35563/user/jenkins/test-data/e9061e95-7d9e-c12a-b652-9e8a1d87f0cf/WALs/5f9d64419733,44215,1733268283642 2024-12-03T23:24:44,763 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [5f9d64419733,40969,1733268283535] 2024-12-03T23:24:44,763 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [5f9d64419733,44411,1733268283680] 2024-12-03T23:24:44,763 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-03T23:24:44,763 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [5f9d64419733,44215,1733268283642] 2024-12-03T23:24:44,767 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35563/user/jenkins/test-data/e9061e95-7d9e-c12a-b652-9e8a1d87f0cf/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T23:24:44,768 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69896038, jitterRate=0.04153212904930115}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-03T23:24:44,771 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1733268284730Initializing all the Stores at 1733268284732 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733268284732Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733268284732Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733268284732Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733268284732Cleaning up temporary data from old regions at 1733268284758 (+26 ms)Region opened successfully at 1733268284770 (+12 ms) 2024-12-03T23:24:44,771 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-03T23:24:44,771 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-03T23:24:44,771 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-03T23:24:44,771 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-03T23:24:44,771 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-03T23:24:44,773 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-03T23:24:44,773 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733268284771Disabling compacts and flushes for region at 1733268284771Disabling writes for close at 1733268284771Writing region close event to WAL at 1733268284772 (+1 ms)Closed at 1733268284772 2024-12-03T23:24:44,776 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-03T23:24:44,776 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-12-03T23:24:44,784 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-12-03T23:24:44,793 INFO [RS:1;5f9d64419733:44215 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-03T23:24:44,793 INFO [RS:2;5f9d64419733:44411 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-03T23:24:44,793 INFO [RS:0;5f9d64419733:40969 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-03T23:24:44,795 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-03T23:24:44,799 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-12-03T23:24:44,808 INFO [RS:1;5f9d64419733:44215 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-03T23:24:44,808 INFO [RS:0;5f9d64419733:40969 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-03T23:24:44,810 INFO [RS:2;5f9d64419733:44411 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-03T23:24:44,815 INFO [RS:2;5f9d64419733:44411 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-03T23:24:44,815 INFO [RS:1;5f9d64419733:44215 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-03T23:24:44,815 INFO [RS:0;5f9d64419733:40969 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-03T23:24:44,815 INFO [RS:1;5f9d64419733:44215 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-03T23:24:44,815 INFO [RS:2;5f9d64419733:44411 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-03T23:24:44,815 INFO [RS:0;5f9d64419733:40969 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-03T23:24:44,817 INFO [RS:1;5f9d64419733:44215 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-03T23:24:44,817 INFO [RS:0;5f9d64419733:40969 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-03T23:24:44,817 INFO [RS:2;5f9d64419733:44411 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-03T23:24:44,824 INFO [RS:2;5f9d64419733:44411 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-03T23:24:44,824 INFO [RS:0;5f9d64419733:40969 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-03T23:24:44,824 INFO [RS:1;5f9d64419733:44215 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-03T23:24:44,825 INFO [RS:2;5f9d64419733:44411 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-03T23:24:44,825 INFO [RS:0;5f9d64419733:40969 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-03T23:24:44,825 INFO [RS:1;5f9d64419733:44215 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-03T23:24:44,826 DEBUG [RS:1;5f9d64419733:44215 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/5f9d64419733:0, corePoolSize=1, maxPoolSize=1 2024-12-03T23:24:44,826 DEBUG [RS:0;5f9d64419733:40969 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/5f9d64419733:0, corePoolSize=1, maxPoolSize=1 2024-12-03T23:24:44,826 DEBUG [RS:2;5f9d64419733:44411 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/5f9d64419733:0, corePoolSize=1, maxPoolSize=1 2024-12-03T23:24:44,826 DEBUG [RS:2;5f9d64419733:44411 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/5f9d64419733:0, corePoolSize=1, maxPoolSize=1 2024-12-03T23:24:44,826 DEBUG [RS:0;5f9d64419733:40969 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/5f9d64419733:0, corePoolSize=1, maxPoolSize=1 2024-12-03T23:24:44,826 DEBUG [RS:1;5f9d64419733:44215 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/5f9d64419733:0, corePoolSize=1, maxPoolSize=1 2024-12-03T23:24:44,826 DEBUG [RS:2;5f9d64419733:44411 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/5f9d64419733:0, corePoolSize=1, maxPoolSize=1 2024-12-03T23:24:44,826 DEBUG [RS:2;5f9d64419733:44411 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/5f9d64419733:0, corePoolSize=1, maxPoolSize=1 2024-12-03T23:24:44,826 DEBUG [RS:1;5f9d64419733:44215 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/5f9d64419733:0, corePoolSize=1, maxPoolSize=1 2024-12-03T23:24:44,826 DEBUG [RS:0;5f9d64419733:40969 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/5f9d64419733:0, corePoolSize=1, maxPoolSize=1 2024-12-03T23:24:44,826 DEBUG [RS:2;5f9d64419733:44411 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/5f9d64419733:0, corePoolSize=1, maxPoolSize=1 2024-12-03T23:24:44,826 DEBUG [RS:1;5f9d64419733:44215 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/5f9d64419733:0, corePoolSize=1, maxPoolSize=1 2024-12-03T23:24:44,826 DEBUG [RS:0;5f9d64419733:40969 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/5f9d64419733:0, corePoolSize=1, maxPoolSize=1 2024-12-03T23:24:44,827 DEBUG [RS:2;5f9d64419733:44411 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/5f9d64419733:0, corePoolSize=2, maxPoolSize=2 2024-12-03T23:24:44,827 DEBUG [RS:2;5f9d64419733:44411 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/5f9d64419733:0, corePoolSize=1, maxPoolSize=1 2024-12-03T23:24:44,827 DEBUG [RS:1;5f9d64419733:44215 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/5f9d64419733:0, corePoolSize=1, maxPoolSize=1 2024-12-03T23:24:44,827 DEBUG [RS:0;5f9d64419733:40969 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/5f9d64419733:0, corePoolSize=1, maxPoolSize=1 2024-12-03T23:24:44,827 DEBUG [RS:2;5f9d64419733:44411 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/5f9d64419733:0, corePoolSize=1, maxPoolSize=1 2024-12-03T23:24:44,827 DEBUG [RS:0;5f9d64419733:40969 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/5f9d64419733:0, corePoolSize=2, maxPoolSize=2 2024-12-03T23:24:44,827 DEBUG [RS:1;5f9d64419733:44215 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/5f9d64419733:0, corePoolSize=2, maxPoolSize=2 2024-12-03T23:24:44,827 DEBUG [RS:2;5f9d64419733:44411 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/5f9d64419733:0, corePoolSize=1, maxPoolSize=1 2024-12-03T23:24:44,827 DEBUG [RS:0;5f9d64419733:40969 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/5f9d64419733:0, corePoolSize=1, maxPoolSize=1 2024-12-03T23:24:44,827 DEBUG [RS:1;5f9d64419733:44215 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/5f9d64419733:0, corePoolSize=1, maxPoolSize=1 2024-12-03T23:24:44,827 DEBUG [RS:2;5f9d64419733:44411 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/5f9d64419733:0, corePoolSize=1, maxPoolSize=1 2024-12-03T23:24:44,827 DEBUG [RS:2;5f9d64419733:44411 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/5f9d64419733:0, corePoolSize=1, maxPoolSize=1 2024-12-03T23:24:44,827 DEBUG [RS:1;5f9d64419733:44215 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/5f9d64419733:0, corePoolSize=1, maxPoolSize=1 2024-12-03T23:24:44,827 DEBUG [RS:0;5f9d64419733:40969 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/5f9d64419733:0, corePoolSize=1, maxPoolSize=1 2024-12-03T23:24:44,827 DEBUG [RS:2;5f9d64419733:44411 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/5f9d64419733:0, corePoolSize=1, maxPoolSize=1 2024-12-03T23:24:44,828 DEBUG [RS:0;5f9d64419733:40969 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/5f9d64419733:0, corePoolSize=1, maxPoolSize=1 2024-12-03T23:24:44,828 DEBUG [RS:1;5f9d64419733:44215 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/5f9d64419733:0, corePoolSize=1, maxPoolSize=1 2024-12-03T23:24:44,828 DEBUG [RS:2;5f9d64419733:44411 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/5f9d64419733:0, corePoolSize=3, maxPoolSize=3 2024-12-03T23:24:44,828 DEBUG [RS:2;5f9d64419733:44411 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/5f9d64419733:0, corePoolSize=3, maxPoolSize=3 2024-12-03T23:24:44,828 DEBUG [RS:1;5f9d64419733:44215 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/5f9d64419733:0, corePoolSize=1, maxPoolSize=1 2024-12-03T23:24:44,828 DEBUG [RS:0;5f9d64419733:40969 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/5f9d64419733:0, corePoolSize=1, maxPoolSize=1 2024-12-03T23:24:44,828 DEBUG [RS:1;5f9d64419733:44215 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/5f9d64419733:0, corePoolSize=1, maxPoolSize=1 2024-12-03T23:24:44,828 DEBUG [RS:0;5f9d64419733:40969 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/5f9d64419733:0, corePoolSize=1, maxPoolSize=1 2024-12-03T23:24:44,828 DEBUG [RS:0;5f9d64419733:40969 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/5f9d64419733:0, corePoolSize=1, maxPoolSize=1 2024-12-03T23:24:44,828 DEBUG [RS:1;5f9d64419733:44215 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/5f9d64419733:0, corePoolSize=1, maxPoolSize=1 2024-12-03T23:24:44,828 DEBUG [RS:0;5f9d64419733:40969 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/5f9d64419733:0, corePoolSize=3, maxPoolSize=3 2024-12-03T23:24:44,828 DEBUG [RS:1;5f9d64419733:44215 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/5f9d64419733:0, corePoolSize=3, maxPoolSize=3 2024-12-03T23:24:44,828 DEBUG [RS:0;5f9d64419733:40969 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/5f9d64419733:0, corePoolSize=3, maxPoolSize=3 2024-12-03T23:24:44,828 DEBUG [RS:1;5f9d64419733:44215 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/5f9d64419733:0, corePoolSize=3, maxPoolSize=3 2024-12-03T23:24:44,832 INFO [RS:2;5f9d64419733:44411 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-03T23:24:44,833 INFO [RS:2;5f9d64419733:44411 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-03T23:24:44,833 INFO [RS:2;5f9d64419733:44411 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-03T23:24:44,833 INFO [RS:2;5f9d64419733:44411 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-03T23:24:44,833 INFO [RS:1;5f9d64419733:44215 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-03T23:24:44,833 INFO [RS:2;5f9d64419733:44411 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-03T23:24:44,833 INFO [RS:0;5f9d64419733:40969 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-03T23:24:44,833 INFO [RS:0;5f9d64419733:40969 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-03T23:24:44,833 INFO [RS:2;5f9d64419733:44411 {}] hbase.ChoreService(168): Chore ScheduledChore name=5f9d64419733,44411,1733268283680-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-03T23:24:44,833 INFO [RS:0;5f9d64419733:40969 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-03T23:24:44,833 INFO [RS:0;5f9d64419733:40969 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-03T23:24:44,833 INFO [RS:0;5f9d64419733:40969 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-03T23:24:44,834 INFO [RS:0;5f9d64419733:40969 {}] hbase.ChoreService(168): Chore ScheduledChore name=5f9d64419733,40969,1733268283535-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-03T23:24:44,834 INFO [RS:1;5f9d64419733:44215 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-03T23:24:44,834 INFO [RS:1;5f9d64419733:44215 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-03T23:24:44,834 INFO [RS:1;5f9d64419733:44215 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-03T23:24:44,834 INFO [RS:1;5f9d64419733:44215 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-03T23:24:44,834 INFO [RS:1;5f9d64419733:44215 {}] hbase.ChoreService(168): Chore ScheduledChore name=5f9d64419733,44215,1733268283642-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-03T23:24:44,863 INFO [RS:1;5f9d64419733:44215 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-03T23:24:44,863 INFO [RS:2;5f9d64419733:44411 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-03T23:24:44,863 INFO [RS:0;5f9d64419733:40969 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-03T23:24:44,865 INFO [RS:0;5f9d64419733:40969 {}] hbase.ChoreService(168): Chore ScheduledChore name=5f9d64419733,40969,1733268283535-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-03T23:24:44,865 INFO [RS:2;5f9d64419733:44411 {}] hbase.ChoreService(168): Chore ScheduledChore name=5f9d64419733,44411,1733268283680-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-03T23:24:44,865 INFO [RS:1;5f9d64419733:44215 {}] hbase.ChoreService(168): Chore ScheduledChore name=5f9d64419733,44215,1733268283642-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-03T23:24:44,865 INFO [RS:0;5f9d64419733:40969 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-03T23:24:44,865 INFO [RS:2;5f9d64419733:44411 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-03T23:24:44,866 INFO [RS:0;5f9d64419733:40969 {}] regionserver.Replication(171): 5f9d64419733,40969,1733268283535 started 2024-12-03T23:24:44,866 INFO [RS:2;5f9d64419733:44411 {}] regionserver.Replication(171): 5f9d64419733,44411,1733268283680 started 2024-12-03T23:24:44,865 INFO [RS:1;5f9d64419733:44215 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-03T23:24:44,866 INFO [RS:1;5f9d64419733:44215 {}] regionserver.Replication(171): 5f9d64419733,44215,1733268283642 started 2024-12-03T23:24:44,884 INFO [RS:1;5f9d64419733:44215 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-03T23:24:44,885 INFO [RS:1;5f9d64419733:44215 {}] regionserver.HRegionServer(1482): Serving as 5f9d64419733,44215,1733268283642, RpcServer on 5f9d64419733/172.17.0.2:44215, sessionid=0x1017aa5055b0002 2024-12-03T23:24:44,885 DEBUG [RS:1;5f9d64419733:44215 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-03T23:24:44,886 DEBUG [RS:1;5f9d64419733:44215 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 5f9d64419733,44215,1733268283642 2024-12-03T23:24:44,886 DEBUG [RS:1;5f9d64419733:44215 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '5f9d64419733,44215,1733268283642' 2024-12-03T23:24:44,886 DEBUG [RS:1;5f9d64419733:44215 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-03T23:24:44,887 DEBUG [RS:1;5f9d64419733:44215 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-03T23:24:44,888 DEBUG [RS:1;5f9d64419733:44215 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-03T23:24:44,888 DEBUG [RS:1;5f9d64419733:44215 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-03T23:24:44,888 DEBUG [RS:1;5f9d64419733:44215 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 5f9d64419733,44215,1733268283642 2024-12-03T23:24:44,888 DEBUG [RS:1;5f9d64419733:44215 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '5f9d64419733,44215,1733268283642' 2024-12-03T23:24:44,888 DEBUG [RS:1;5f9d64419733:44215 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-03T23:24:44,889 DEBUG [RS:1;5f9d64419733:44215 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-03T23:24:44,889 DEBUG [RS:1;5f9d64419733:44215 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-03T23:24:44,889 INFO [RS:1;5f9d64419733:44215 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-03T23:24:44,890 INFO [RS:1;5f9d64419733:44215 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-03T23:24:44,890 INFO [RS:0;5f9d64419733:40969 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-03T23:24:44,890 INFO [RS:2;5f9d64419733:44411 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-03T23:24:44,891 INFO [RS:0;5f9d64419733:40969 {}] regionserver.HRegionServer(1482): Serving as 5f9d64419733,40969,1733268283535, RpcServer on 5f9d64419733/172.17.0.2:40969, sessionid=0x1017aa5055b0001 2024-12-03T23:24:44,891 INFO [RS:2;5f9d64419733:44411 {}] regionserver.HRegionServer(1482): Serving as 5f9d64419733,44411,1733268283680, RpcServer on 5f9d64419733/172.17.0.2:44411, sessionid=0x1017aa5055b0003 2024-12-03T23:24:44,891 DEBUG [RS:0;5f9d64419733:40969 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-03T23:24:44,891 DEBUG [RS:2;5f9d64419733:44411 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-03T23:24:44,891 DEBUG [RS:2;5f9d64419733:44411 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 5f9d64419733,44411,1733268283680 2024-12-03T23:24:44,891 DEBUG [RS:0;5f9d64419733:40969 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 5f9d64419733,40969,1733268283535 2024-12-03T23:24:44,891 DEBUG [RS:0;5f9d64419733:40969 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '5f9d64419733,40969,1733268283535' 2024-12-03T23:24:44,891 DEBUG [RS:2;5f9d64419733:44411 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '5f9d64419733,44411,1733268283680' 2024-12-03T23:24:44,891 DEBUG [RS:2;5f9d64419733:44411 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-03T23:24:44,891 DEBUG [RS:0;5f9d64419733:40969 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-03T23:24:44,893 DEBUG [RS:2;5f9d64419733:44411 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-03T23:24:44,893 DEBUG [RS:0;5f9d64419733:40969 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-03T23:24:44,894 DEBUG [RS:2;5f9d64419733:44411 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-03T23:24:44,894 DEBUG [RS:0;5f9d64419733:40969 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-03T23:24:44,894 DEBUG [RS:2;5f9d64419733:44411 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-03T23:24:44,894 DEBUG [RS:0;5f9d64419733:40969 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-03T23:24:44,894 DEBUG [RS:2;5f9d64419733:44411 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 5f9d64419733,44411,1733268283680 2024-12-03T23:24:44,894 DEBUG [RS:0;5f9d64419733:40969 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 5f9d64419733,40969,1733268283535 2024-12-03T23:24:44,894 DEBUG [RS:2;5f9d64419733:44411 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '5f9d64419733,44411,1733268283680' 2024-12-03T23:24:44,894 DEBUG [RS:0;5f9d64419733:40969 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '5f9d64419733,40969,1733268283535' 2024-12-03T23:24:44,894 DEBUG [RS:2;5f9d64419733:44411 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-03T23:24:44,894 DEBUG [RS:0;5f9d64419733:40969 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-03T23:24:44,895 DEBUG [RS:2;5f9d64419733:44411 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-03T23:24:44,895 DEBUG [RS:0;5f9d64419733:40969 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-03T23:24:44,895 DEBUG [RS:2;5f9d64419733:44411 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-03T23:24:44,895 DEBUG [RS:0;5f9d64419733:40969 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-03T23:24:44,896 INFO [RS:2;5f9d64419733:44411 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-03T23:24:44,896 INFO [RS:2;5f9d64419733:44411 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-03T23:24:44,896 INFO [RS:0;5f9d64419733:40969 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-03T23:24:44,896 INFO [RS:0;5f9d64419733:40969 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-03T23:24:44,950 WARN [5f9d64419733:41995 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-03T23:24:44,995 INFO [RS:1;5f9d64419733:44215 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-03T23:24:44,997 INFO [RS:0;5f9d64419733:40969 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-03T23:24:44,997 INFO [RS:2;5f9d64419733:44411 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-03T23:24:44,999 INFO [RS:1;5f9d64419733:44215 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=5f9d64419733%2C44215%2C1733268283642, suffix=, logDir=hdfs://localhost:35563/user/jenkins/test-data/e9061e95-7d9e-c12a-b652-9e8a1d87f0cf/WALs/5f9d64419733,44215,1733268283642, archiveDir=hdfs://localhost:35563/user/jenkins/test-data/e9061e95-7d9e-c12a-b652-9e8a1d87f0cf/oldWALs, maxLogs=32 2024-12-03T23:24:45,000 INFO [RS:0;5f9d64419733:40969 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=5f9d64419733%2C40969%2C1733268283535, suffix=, logDir=hdfs://localhost:35563/user/jenkins/test-data/e9061e95-7d9e-c12a-b652-9e8a1d87f0cf/WALs/5f9d64419733,40969,1733268283535, archiveDir=hdfs://localhost:35563/user/jenkins/test-data/e9061e95-7d9e-c12a-b652-9e8a1d87f0cf/oldWALs, maxLogs=32 2024-12-03T23:24:45,000 INFO [RS:2;5f9d64419733:44411 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=5f9d64419733%2C44411%2C1733268283680, suffix=, logDir=hdfs://localhost:35563/user/jenkins/test-data/e9061e95-7d9e-c12a-b652-9e8a1d87f0cf/WALs/5f9d64419733,44411,1733268283680, archiveDir=hdfs://localhost:35563/user/jenkins/test-data/e9061e95-7d9e-c12a-b652-9e8a1d87f0cf/oldWALs, maxLogs=32 2024-12-03T23:24:45,026 DEBUG [RS:1;5f9d64419733:44215 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/e9061e95-7d9e-c12a-b652-9e8a1d87f0cf/WALs/5f9d64419733,44215,1733268283642/5f9d64419733%2C44215%2C1733268283642.1733268285003, exclude list is [], retry=0 2024-12-03T23:24:45,026 DEBUG [RS:2;5f9d64419733:44411 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/e9061e95-7d9e-c12a-b652-9e8a1d87f0cf/WALs/5f9d64419733,44411,1733268283680/5f9d64419733%2C44411%2C1733268283680.1733268285003, exclude list is [], retry=0 2024-12-03T23:24:45,026 DEBUG [RS:0;5f9d64419733:40969 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/e9061e95-7d9e-c12a-b652-9e8a1d87f0cf/WALs/5f9d64419733,40969,1733268283535/5f9d64419733%2C40969%2C1733268283535.1733268285003, exclude list is [], retry=0 2024-12-03T23:24:45,032 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45491,DS-d55e5535-c0ce-4dc2-a45c-8e70eb010e56,DISK] 2024-12-03T23:24:45,033 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41487,DS-459d168f-61f7-403b-9139-91eca369c54e,DISK] 2024-12-03T23:24:45,033 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39829,DS-e6270720-ef81-458e-b5d6-959d16c4c378,DISK] 2024-12-03T23:24:45,033 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41487,DS-459d168f-61f7-403b-9139-91eca369c54e,DISK] 2024-12-03T23:24:45,033 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39829,DS-e6270720-ef81-458e-b5d6-959d16c4c378,DISK] 2024-12-03T23:24:45,034 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41487,DS-459d168f-61f7-403b-9139-91eca369c54e,DISK] 2024-12-03T23:24:45,034 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45491,DS-d55e5535-c0ce-4dc2-a45c-8e70eb010e56,DISK] 2024-12-03T23:24:45,034 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45491,DS-d55e5535-c0ce-4dc2-a45c-8e70eb010e56,DISK] 2024-12-03T23:24:45,034 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39829,DS-e6270720-ef81-458e-b5d6-959d16c4c378,DISK] 2024-12-03T23:24:45,040 INFO [RS:2;5f9d64419733:44411 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/e9061e95-7d9e-c12a-b652-9e8a1d87f0cf/WALs/5f9d64419733,44411,1733268283680/5f9d64419733%2C44411%2C1733268283680.1733268285003 2024-12-03T23:24:45,045 DEBUG [RS:2;5f9d64419733:44411 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36361:36361),(127.0.0.1/127.0.0.1:34451:34451),(127.0.0.1/127.0.0.1:38065:38065)] 2024-12-03T23:24:45,045 INFO [RS:1;5f9d64419733:44215 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/e9061e95-7d9e-c12a-b652-9e8a1d87f0cf/WALs/5f9d64419733,44215,1733268283642/5f9d64419733%2C44215%2C1733268283642.1733268285003 2024-12-03T23:24:45,045 INFO [RS:0;5f9d64419733:40969 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/e9061e95-7d9e-c12a-b652-9e8a1d87f0cf/WALs/5f9d64419733,40969,1733268283535/5f9d64419733%2C40969%2C1733268283535.1733268285003 2024-12-03T23:24:45,048 DEBUG [RS:1;5f9d64419733:44215 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36361:36361),(127.0.0.1/127.0.0.1:34451:34451),(127.0.0.1/127.0.0.1:38065:38065)] 2024-12-03T23:24:45,049 DEBUG [RS:0;5f9d64419733:40969 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:38065:38065),(127.0.0.1/127.0.0.1:36361:36361),(127.0.0.1/127.0.0.1:34451:34451)] 2024-12-03T23:24:45,203 DEBUG [5f9d64419733:41995 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=3, allServersCount=3 2024-12-03T23:24:45,211 DEBUG [5f9d64419733:41995 {}] balancer.BalancerClusterState(204): Hosts are {5f9d64419733=0} racks are {/default-rack=0} 2024-12-03T23:24:45,218 DEBUG [5f9d64419733:41995 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-12-03T23:24:45,218 DEBUG [5f9d64419733:41995 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-12-03T23:24:45,218 DEBUG [5f9d64419733:41995 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-12-03T23:24:45,218 DEBUG [5f9d64419733:41995 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-03T23:24:45,218 DEBUG [5f9d64419733:41995 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-03T23:24:45,218 DEBUG [5f9d64419733:41995 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-12-03T23:24:45,218 INFO [5f9d64419733:41995 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-03T23:24:45,218 INFO [5f9d64419733:41995 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-03T23:24:45,218 INFO [5f9d64419733:41995 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-12-03T23:24:45,218 DEBUG [5f9d64419733:41995 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-12-03T23:24:45,226 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=5f9d64419733,40969,1733268283535 2024-12-03T23:24:45,231 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 5f9d64419733,40969,1733268283535, state=OPENING 2024-12-03T23:24:45,237 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-12-03T23:24:45,239 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40969-0x1017aa5055b0001, quorum=127.0.0.1:51247, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T23:24:45,239 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41995-0x1017aa5055b0000, quorum=127.0.0.1:51247, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T23:24:45,239 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44215-0x1017aa5055b0002, quorum=127.0.0.1:51247, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T23:24:45,239 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44411-0x1017aa5055b0003, quorum=127.0.0.1:51247, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T23:24:45,239 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T23:24:45,240 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T23:24:45,240 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T23:24:45,240 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T23:24:45,241 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-03T23:24:45,243 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=5f9d64419733,40969,1733268283535}] 2024-12-03T23:24:45,420 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-03T23:24:45,422 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49485, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-03T23:24:45,435 INFO [RS_OPEN_META-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-12-03T23:24:45,436 INFO [RS_OPEN_META-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-03T23:24:45,436 INFO [RS_OPEN_META-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-12-03T23:24:45,439 INFO [RS_OPEN_META-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=5f9d64419733%2C40969%2C1733268283535.meta, suffix=.meta, logDir=hdfs://localhost:35563/user/jenkins/test-data/e9061e95-7d9e-c12a-b652-9e8a1d87f0cf/WALs/5f9d64419733,40969,1733268283535, archiveDir=hdfs://localhost:35563/user/jenkins/test-data/e9061e95-7d9e-c12a-b652-9e8a1d87f0cf/oldWALs, maxLogs=32 2024-12-03T23:24:45,456 DEBUG [RS_OPEN_META-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/e9061e95-7d9e-c12a-b652-9e8a1d87f0cf/WALs/5f9d64419733,40969,1733268283535/5f9d64419733%2C40969%2C1733268283535.meta.1733268285441.meta, exclude list is [], retry=0 2024-12-03T23:24:45,461 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39829,DS-e6270720-ef81-458e-b5d6-959d16c4c378,DISK] 2024-12-03T23:24:45,461 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45491,DS-d55e5535-c0ce-4dc2-a45c-8e70eb010e56,DISK] 2024-12-03T23:24:45,461 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41487,DS-459d168f-61f7-403b-9139-91eca369c54e,DISK] 2024-12-03T23:24:45,464 INFO [RS_OPEN_META-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/e9061e95-7d9e-c12a-b652-9e8a1d87f0cf/WALs/5f9d64419733,40969,1733268283535/5f9d64419733%2C40969%2C1733268283535.meta.1733268285441.meta 2024-12-03T23:24:45,465 DEBUG [RS_OPEN_META-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36361:36361),(127.0.0.1/127.0.0.1:38065:38065),(127.0.0.1/127.0.0.1:34451:34451)] 2024-12-03T23:24:45,465 DEBUG [RS_OPEN_META-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-12-03T23:24:45,467 DEBUG [RS_OPEN_META-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-12-03T23:24:45,470 DEBUG [RS_OPEN_META-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-12-03T23:24:45,475 INFO [RS_OPEN_META-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-12-03T23:24:45,479 DEBUG [RS_OPEN_META-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-12-03T23:24:45,480 DEBUG [RS_OPEN_META-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T23:24:45,480 DEBUG [RS_OPEN_META-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-12-03T23:24:45,480 DEBUG [RS_OPEN_META-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-12-03T23:24:45,483 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-03T23:24:45,484 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-03T23:24:45,484 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T23:24:45,485 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-03T23:24:45,485 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-03T23:24:45,486 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-03T23:24:45,487 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T23:24:45,487 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-03T23:24:45,487 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-03T23:24:45,489 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-03T23:24:45,489 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T23:24:45,489 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-03T23:24:45,490 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-03T23:24:45,491 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-03T23:24:45,491 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T23:24:45,492 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-03T23:24:45,492 DEBUG [RS_OPEN_META-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-03T23:24:45,493 DEBUG [RS_OPEN_META-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35563/user/jenkins/test-data/e9061e95-7d9e-c12a-b652-9e8a1d87f0cf/data/hbase/meta/1588230740 2024-12-03T23:24:45,495 DEBUG [RS_OPEN_META-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35563/user/jenkins/test-data/e9061e95-7d9e-c12a-b652-9e8a1d87f0cf/data/hbase/meta/1588230740 2024-12-03T23:24:45,498 DEBUG [RS_OPEN_META-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-03T23:24:45,498 DEBUG [RS_OPEN_META-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-03T23:24:45,499 DEBUG [RS_OPEN_META-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-03T23:24:45,501 DEBUG [RS_OPEN_META-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-03T23:24:45,503 INFO [RS_OPEN_META-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67680781, jitterRate=0.00852222740650177}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-03T23:24:45,503 DEBUG [RS_OPEN_META-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-12-03T23:24:45,505 DEBUG [RS_OPEN_META-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1733268285480Writing region info on filesystem at 1733268285480Initializing all the Stores at 1733268285482 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733268285482Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733268285483 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733268285483Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733268285483Cleaning up temporary data from old regions at 1733268285498 (+15 ms)Running coprocessor post-open hooks at 1733268285503 (+5 ms)Region opened successfully at 1733268285505 (+2 ms) 2024-12-03T23:24:45,515 INFO [RS_OPEN_META-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733268285410 2024-12-03T23:24:45,530 DEBUG [RS_OPEN_META-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-12-03T23:24:45,531 INFO [RS_OPEN_META-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-12-03T23:24:45,532 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=5f9d64419733,40969,1733268283535 2024-12-03T23:24:45,534 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 5f9d64419733,40969,1733268283535, state=OPEN 2024-12-03T23:24:45,538 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41995-0x1017aa5055b0000, quorum=127.0.0.1:51247, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-03T23:24:45,538 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40969-0x1017aa5055b0001, quorum=127.0.0.1:51247, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-03T23:24:45,538 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44411-0x1017aa5055b0003, quorum=127.0.0.1:51247, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-03T23:24:45,538 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44215-0x1017aa5055b0002, quorum=127.0.0.1:51247, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-03T23:24:45,538 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T23:24:45,538 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T23:24:45,538 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T23:24:45,538 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T23:24:45,539 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=5f9d64419733,40969,1733268283535 2024-12-03T23:24:45,544 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-12-03T23:24:45,544 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=5f9d64419733,40969,1733268283535 in 296 msec 2024-12-03T23:24:45,551 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-12-03T23:24:45,551 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 762 msec 2024-12-03T23:24:45,553 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-03T23:24:45,553 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-12-03T23:24:45,575 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T23:24:45,577 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5f9d64419733,40969,1733268283535, seqNum=-1] 2024-12-03T23:24:45,632 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T23:24:45,638 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56655, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T23:24:45,661 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.1040 sec 2024-12-03T23:24:45,661 INFO [master/5f9d64419733:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733268285661, completionTime=-1 2024-12-03T23:24:45,664 INFO [master/5f9d64419733:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=3; waited=0ms, expected min=3 server(s), max=3 server(s), master is running 2024-12-03T23:24:45,664 DEBUG [master/5f9d64419733:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-03T23:24:45,691 INFO [master/5f9d64419733:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=3 2024-12-03T23:24:45,691 INFO [master/5f9d64419733:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733268345691 2024-12-03T23:24:45,691 INFO [master/5f9d64419733:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733268405691 2024-12-03T23:24:45,691 INFO [master/5f9d64419733:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 27 msec 2024-12-03T23:24:45,693 DEBUG [master/5f9d64419733:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-12-03T23:24:45,700 INFO [master/5f9d64419733:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=5f9d64419733,41995,1733268282840-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-03T23:24:45,700 INFO [master/5f9d64419733:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=5f9d64419733,41995,1733268282840-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-03T23:24:45,700 INFO [master/5f9d64419733:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=5f9d64419733,41995,1733268282840-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-03T23:24:45,702 INFO [master/5f9d64419733:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-5f9d64419733:41995, period=300000, unit=MILLISECONDS is enabled. 2024-12-03T23:24:45,702 INFO [master/5f9d64419733:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-12-03T23:24:45,703 INFO [master/5f9d64419733:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-12-03T23:24:45,709 DEBUG [master/5f9d64419733:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-12-03T23:24:45,731 INFO [master/5f9d64419733:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.969sec 2024-12-03T23:24:45,732 INFO [master/5f9d64419733:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-12-03T23:24:45,734 INFO [master/5f9d64419733:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-12-03T23:24:45,735 INFO [master/5f9d64419733:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-12-03T23:24:45,735 INFO [master/5f9d64419733:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-12-03T23:24:45,735 INFO [master/5f9d64419733:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-12-03T23:24:45,736 INFO [master/5f9d64419733:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=5f9d64419733,41995,1733268282840-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-03T23:24:45,737 INFO [master/5f9d64419733:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=5f9d64419733,41995,1733268282840-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-12-03T23:24:45,741 DEBUG [master/5f9d64419733:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-12-03T23:24:45,742 INFO [master/5f9d64419733:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-12-03T23:24:45,742 INFO [master/5f9d64419733:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=5f9d64419733,41995,1733268282840-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-03T23:24:45,820 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@38ad9a8d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T23:24:45,821 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5f9d64419733,41995,-1 for getting cluster id 2024-12-03T23:24:45,824 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T23:24:45,833 DEBUG [HMaster-EventLoopGroup-2-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c861c93d-86c4-4933-988c-7abbfe26c2c0' 2024-12-03T23:24:45,836 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T23:24:45,836 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c861c93d-86c4-4933-988c-7abbfe26c2c0" 2024-12-03T23:24:45,839 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@666c4a1b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T23:24:45,839 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5f9d64419733,41995,-1] 2024-12-03T23:24:45,841 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T23:24:45,843 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T23:24:45,844 INFO [HMaster-EventLoopGroup-2-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55704, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T23:24:45,846 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@20190245, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T23:24:45,847 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T23:24:45,854 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5f9d64419733,40969,1733268283535, seqNum=-1] 2024-12-03T23:24:45,855 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T23:24:45,857 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34998, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T23:24:45,877 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=5f9d64419733,41995,1733268282840 2024-12-03T23:24:45,877 INFO [Time-limited test {}] wal.AbstractTestWALReplay(147): hbase.rootdir=hdfs://localhost:35563/hbase 2024-12-03T23:24:45,892 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayCompressed#testNameConflictWhenSplit0 Thread=368, OpenFileDescriptor=611, MaxFileDescriptor=1048576, SystemLoadAverage=239, ProcessCount=11, AvailableMemoryMB=8192 2024-12-03T23:24:45,910 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-03T23:24:45,913 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-03T23:24:45,914 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-03T23:24:45,918 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-34669795, suffix=, logDir=hdfs://localhost:35563/hbase/WALs/hregion-34669795, archiveDir=hdfs://localhost:35563/hbase/oldWALs, maxLogs=32 2024-12-03T23:24:45,934 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-34669795/hregion-34669795.1733268285920, exclude list is [], retry=0 2024-12-03T23:24:45,939 DEBUG [AsyncFSWAL-8-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45491,DS-d55e5535-c0ce-4dc2-a45c-8e70eb010e56,DISK] 2024-12-03T23:24:45,939 DEBUG [AsyncFSWAL-8-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39829,DS-e6270720-ef81-458e-b5d6-959d16c4c378,DISK] 2024-12-03T23:24:45,940 DEBUG [AsyncFSWAL-8-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41487,DS-459d168f-61f7-403b-9139-91eca369c54e,DISK] 2024-12-03T23:24:45,943 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-34669795/hregion-34669795.1733268285920 2024-12-03T23:24:45,944 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34451:34451),(127.0.0.1/127.0.0.1:38065:38065),(127.0.0.1/127.0.0.1:36361:36361)] 2024-12-03T23:24:45,944 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 998b84cb1ff32c517b3d84e9f464853f, NAME => 'testReplayEditsWrittenIntoWAL,,1733268285910.998b84cb1ff32c517b3d84e9f464853f.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenIntoWAL', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35563/hbase 2024-12-03T23:24:45,959 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41487 is added to blk_1073741838_1014 (size=64) 2024-12-03T23:24:45,959 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39829 is added to blk_1073741838_1014 (size=64) 2024-12-03T23:24:45,960 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45491 is added to blk_1073741838_1014 (size=64) 2024-12-03T23:24:45,961 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1733268285910.998b84cb1ff32c517b3d84e9f464853f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T23:24:45,963 INFO [StoreOpener-998b84cb1ff32c517b3d84e9f464853f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 998b84cb1ff32c517b3d84e9f464853f 2024-12-03T23:24:45,966 INFO [StoreOpener-998b84cb1ff32c517b3d84e9f464853f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 998b84cb1ff32c517b3d84e9f464853f columnFamilyName a 2024-12-03T23:24:45,966 DEBUG [StoreOpener-998b84cb1ff32c517b3d84e9f464853f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T23:24:45,967 INFO [StoreOpener-998b84cb1ff32c517b3d84e9f464853f-1 {}] regionserver.HStore(327): Store=998b84cb1ff32c517b3d84e9f464853f/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T23:24:45,967 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 998b84cb1ff32c517b3d84e9f464853f 2024-12-03T23:24:45,968 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35563/hbase/data/default/testReplayEditsWrittenIntoWAL/998b84cb1ff32c517b3d84e9f464853f 2024-12-03T23:24:45,969 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35563/hbase/data/default/testReplayEditsWrittenIntoWAL/998b84cb1ff32c517b3d84e9f464853f 2024-12-03T23:24:45,969 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 998b84cb1ff32c517b3d84e9f464853f 2024-12-03T23:24:45,969 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 998b84cb1ff32c517b3d84e9f464853f 2024-12-03T23:24:45,972 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 998b84cb1ff32c517b3d84e9f464853f 2024-12-03T23:24:45,976 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35563/hbase/data/default/testReplayEditsWrittenIntoWAL/998b84cb1ff32c517b3d84e9f464853f/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T23:24:45,977 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 998b84cb1ff32c517b3d84e9f464853f; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71418917, jitterRate=0.06422479450702667}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-03T23:24:45,979 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 998b84cb1ff32c517b3d84e9f464853f: Writing region info on filesystem at 1733268285961Initializing all the Stores at 1733268285963 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733268285963Cleaning up temporary data from old regions at 1733268285969 (+6 ms)Region opened successfully at 1733268285979 (+10 ms) 2024-12-03T23:24:45,979 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 998b84cb1ff32c517b3d84e9f464853f, disabling compactions & flushes 2024-12-03T23:24:45,979 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1733268285910.998b84cb1ff32c517b3d84e9f464853f. 2024-12-03T23:24:45,979 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1733268285910.998b84cb1ff32c517b3d84e9f464853f. 2024-12-03T23:24:45,979 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1733268285910.998b84cb1ff32c517b3d84e9f464853f. after waiting 0 ms 2024-12-03T23:24:45,979 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1733268285910.998b84cb1ff32c517b3d84e9f464853f. 2024-12-03T23:24:45,980 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1733268285910.998b84cb1ff32c517b3d84e9f464853f. 2024-12-03T23:24:45,980 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 998b84cb1ff32c517b3d84e9f464853f: Waiting for close lock at 1733268285979Disabling compacts and flushes for region at 1733268285979Disabling writes for close at 1733268285979Writing region close event to WAL at 1733268285980 (+1 ms)Closed at 1733268285980 2024-12-03T23:24:45,984 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39829 is added to blk_1073741837_1013 (size=93) 2024-12-03T23:24:45,984 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45491 is added to blk_1073741837_1013 (size=93) 2024-12-03T23:24:45,985 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41487 is added to blk_1073741837_1013 (size=93) 2024-12-03T23:24:45,991 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-03T23:24:45,991 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-34669795:(num 1733268285920) 2024-12-03T23:24:45,993 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-12-03T23:24:46,008 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41487 is added to blk_1073741839_1015 (size=276) 2024-12-03T23:24:46,008 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45491 is added to blk_1073741839_1015 (size=276) 2024-12-03T23:24:46,009 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39829 is added to blk_1073741839_1015 (size=276) 2024-12-03T23:24:46,010 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-12-03T23:24:46,024 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39829 is added to blk_1073741840_1016 (size=230) 2024-12-03T23:24:46,024 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45491 is added to blk_1073741840_1016 (size=230) 2024-12-03T23:24:46,025 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41487 is added to blk_1073741840_1016 (size=230) 2024-12-03T23:24:46,047 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:35563/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733268285908/wal-1, size=276 (276bytes) 2024-12-03T23:24:46,048 DEBUG [Time-limited test {}] util.RecoverLeaseFSUtils(47): Initialize RecoverLeaseFSUtils 2024-12-03T23:24:46,048 DEBUG [Time-limited test {}] util.RecoverLeaseFSUtils(59): set recoverLeaseMethod to org.apache.hadoop.fs.LeaseRecoverable.recoverLease() 2024-12-03T23:24:46,048 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:35563/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733268285908/wal-1 2024-12-03T23:24:46,054 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:35563/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733268285908/wal-1 after 4ms 2024-12-03T23:24:46,060 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:35563/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733268285908/wal-1: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-03T23:24:46,061 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:35563/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733268285908/wal-1 took 15ms 2024-12-03T23:24:46,065 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:35563/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733268285908/wal-1 so closing down 2024-12-03T23:24:46,068 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal-1.temp 2024-12-03T23:24:46,069 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:35563/hbase/data/default/testReplayEditsWrittenIntoWAL/998b84cb1ff32c517b3d84e9f464853f/recovered.edits/0000000000000000001-wal-1.temp 2024-12-03T23:24:46,070 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-03T23:24:46,070 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-03T23:24:46,082 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41487 is added to blk_1073741841_1017 (size=276) 2024-12-03T23:24:46,082 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45491 is added to blk_1073741841_1017 (size=276) 2024-12-03T23:24:46,083 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39829 is added to blk_1073741841_1017 (size=276) 2024-12-03T23:24:46,085 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:35563/hbase/data/default/testReplayEditsWrittenIntoWAL/998b84cb1ff32c517b3d84e9f464853f/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) 2024-12-03T23:24:46,087 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:35563/hbase/data/default/testReplayEditsWrittenIntoWAL/998b84cb1ff32c517b3d84e9f464853f/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:35563/hbase/data/default/testReplayEditsWrittenIntoWAL/998b84cb1ff32c517b3d84e9f464853f/recovered.edits/0000000000000000002 2024-12-03T23:24:46,091 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 2 edits across 1 Regions in 26 ms; skipped=0; WAL=hdfs://localhost:35563/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733268285908/wal-1, size=276, length=276, corrupted=false, cancelled=false 2024-12-03T23:24:46,092 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:35563/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733268285908/wal-1, journal: Splitting hdfs://localhost:35563/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733268285908/wal-1, size=276 (276bytes) at 1733268286047Finishing writing output for hdfs://localhost:35563/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733268285908/wal-1 so closing down at 1733268286065 (+18 ms)Creating recovered edits writer path=hdfs://localhost:35563/hbase/data/default/testReplayEditsWrittenIntoWAL/998b84cb1ff32c517b3d84e9f464853f/recovered.edits/0000000000000000001-wal-1.temp at 1733268286069 (+4 ms)3 split writer threads finished at 1733268286070 (+1 ms)Closed recovered edits writer path=hdfs://localhost:35563/hbase/data/default/testReplayEditsWrittenIntoWAL/998b84cb1ff32c517b3d84e9f464853f/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) at 1733268286085 (+15 ms)Rename recovered edits hdfs://localhost:35563/hbase/data/default/testReplayEditsWrittenIntoWAL/998b84cb1ff32c517b3d84e9f464853f/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:35563/hbase/data/default/testReplayEditsWrittenIntoWAL/998b84cb1ff32c517b3d84e9f464853f/recovered.edits/0000000000000000002 at 1733268286087 (+2 ms)Processed 2 edits across 1 Regions in 26 ms; skipped=0; WAL=hdfs://localhost:35563/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733268285908/wal-1, size=276, length=276, corrupted=false, cancelled=false at 1733268286091 (+4 ms) 2024-12-03T23:24:46,106 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:35563/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733268285908/wal-2, size=230 (230bytes) 2024-12-03T23:24:46,106 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:35563/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733268285908/wal-2 2024-12-03T23:24:46,107 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:35563/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733268285908/wal-2 after 1ms 2024-12-03T23:24:46,110 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:35563/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733268285908/wal-2: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-03T23:24:46,111 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:35563/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733268285908/wal-2 took 5ms 2024-12-03T23:24:46,113 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:35563/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733268285908/wal-2 so closing down 2024-12-03T23:24:46,113 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-03T23:24:46,116 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000002-wal-2.temp 2024-12-03T23:24:46,118 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:35563/hbase/data/default/testReplayEditsWrittenIntoWAL/998b84cb1ff32c517b3d84e9f464853f/recovered.edits/0000000000000000002-wal-2.temp 2024-12-03T23:24:46,118 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-03T23:24:46,126 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41487 is added to blk_1073741842_1018 (size=230) 2024-12-03T23:24:46,126 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39829 is added to blk_1073741842_1018 (size=230) 2024-12-03T23:24:46,128 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:35563/hbase/data/default/testReplayEditsWrittenIntoWAL/998b84cb1ff32c517b3d84e9f464853f/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-12-03T23:24:46,128 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45491 is added to blk_1073741842_1018 (size=230) 2024-12-03T23:24:46,133 DEBUG [split-log-closeStream-pool-0 {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:35563/hbase/data/default/testReplayEditsWrittenIntoWAL/998b84cb1ff32c517b3d84e9f464853f/recovered.edits/0000000000000000002: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-03T23:24:46,136 WARN [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(184): Found existing old edits file and we have less entries. Deleting hdfs://localhost:35563/hbase/data/default/testReplayEditsWrittenIntoWAL/998b84cb1ff32c517b3d84e9f464853f/recovered.edits/0000000000000000002-wal-2.temp, length=230 2024-12-03T23:24:46,137 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 1 edits across 1 Regions in 26 ms; skipped=0; WAL=hdfs://localhost:35563/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733268285908/wal-2, size=230, length=230, corrupted=false, cancelled=false 2024-12-03T23:24:46,138 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:35563/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733268285908/wal-2, journal: Splitting hdfs://localhost:35563/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733268285908/wal-2, size=230 (230bytes) at 1733268286106Finishing writing output for hdfs://localhost:35563/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733268285908/wal-2 so closing down at 1733268286113 (+7 ms)Creating recovered edits writer path=hdfs://localhost:35563/hbase/data/default/testReplayEditsWrittenIntoWAL/998b84cb1ff32c517b3d84e9f464853f/recovered.edits/0000000000000000002-wal-2.temp at 1733268286118 (+5 ms)3 split writer threads finished at 1733268286118Closed recovered edits writer path=hdfs://localhost:35563/hbase/data/default/testReplayEditsWrittenIntoWAL/998b84cb1ff32c517b3d84e9f464853f/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1733268286128 (+10 ms)Processed 1 edits across 1 Regions in 26 ms; skipped=0; WAL=hdfs://localhost:35563/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733268285908/wal-2, size=230, length=230, corrupted=false, cancelled=false at 1733268286137 (+9 ms) 2024-12-03T23:24:46,138 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-03T23:24:46,140 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:35563/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733268285908, archiveDir=hdfs://localhost:35563/hbase/oldWALs, maxLogs=32 2024-12-03T23:24:46,158 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testnameconflictwhensplit0-manual,16010,1733268285908/wal.1733268286142, exclude list is [], retry=0 2024-12-03T23:24:46,162 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41487,DS-459d168f-61f7-403b-9139-91eca369c54e,DISK] 2024-12-03T23:24:46,163 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45491,DS-d55e5535-c0ce-4dc2-a45c-8e70eb010e56,DISK] 2024-12-03T23:24:46,163 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39829,DS-e6270720-ef81-458e-b5d6-959d16c4c378,DISK] 2024-12-03T23:24:46,165 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testnameconflictwhensplit0-manual,16010,1733268285908/wal.1733268286142 2024-12-03T23:24:46,166 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36361:36361),(127.0.0.1/127.0.0.1:34451:34451),(127.0.0.1/127.0.0.1:38065:38065)] 2024-12-03T23:24:46,166 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 998b84cb1ff32c517b3d84e9f464853f, NAME => 'testReplayEditsWrittenIntoWAL,,1733268285910.998b84cb1ff32c517b3d84e9f464853f.', STARTKEY => '', ENDKEY => ''} 2024-12-03T23:24:46,166 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1733268285910.998b84cb1ff32c517b3d84e9f464853f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T23:24:46,166 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 998b84cb1ff32c517b3d84e9f464853f 2024-12-03T23:24:46,166 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 998b84cb1ff32c517b3d84e9f464853f 2024-12-03T23:24:46,168 INFO [StoreOpener-998b84cb1ff32c517b3d84e9f464853f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 998b84cb1ff32c517b3d84e9f464853f 2024-12-03T23:24:46,170 INFO [StoreOpener-998b84cb1ff32c517b3d84e9f464853f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 998b84cb1ff32c517b3d84e9f464853f columnFamilyName a 2024-12-03T23:24:46,170 DEBUG [StoreOpener-998b84cb1ff32c517b3d84e9f464853f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T23:24:46,171 INFO [StoreOpener-998b84cb1ff32c517b3d84e9f464853f-1 {}] regionserver.HStore(327): Store=998b84cb1ff32c517b3d84e9f464853f/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T23:24:46,171 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 998b84cb1ff32c517b3d84e9f464853f 2024-12-03T23:24:46,172 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35563/hbase/data/default/testReplayEditsWrittenIntoWAL/998b84cb1ff32c517b3d84e9f464853f 2024-12-03T23:24:46,175 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:35563/hbase/data/default/testReplayEditsWrittenIntoWAL/998b84cb1ff32c517b3d84e9f464853f 2024-12-03T23:24:46,176 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:35563/hbase/data/default/testReplayEditsWrittenIntoWAL/998b84cb1ff32c517b3d84e9f464853f/recovered.edits/0000000000000000002 2024-12-03T23:24:46,179 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:35563/hbase/data/default/testReplayEditsWrittenIntoWAL/998b84cb1ff32c517b3d84e9f464853f/recovered.edits/0000000000000000002: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-03T23:24:46,185 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 2, skipped 0, firstSequenceIdInLog=1, maxSequenceIdInLog=2, path=hdfs://localhost:35563/hbase/data/default/testReplayEditsWrittenIntoWAL/998b84cb1ff32c517b3d84e9f464853f/recovered.edits/0000000000000000002 2024-12-03T23:24:46,188 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 998b84cb1ff32c517b3d84e9f464853f 1/1 column families, dataSize=108 B heapSize=512 B 2024-12-03T23:24:46,245 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35563/hbase/data/default/testReplayEditsWrittenIntoWAL/998b84cb1ff32c517b3d84e9f464853f/.tmp/a/c6b33ea0722a4396864d9726b7dd2969 is 58, key is testReplayEditsWrittenIntoWAL/a:1/1733268285991/Put/seqid=0 2024-12-03T23:24:46,259 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41487 is added to blk_1073741844_1020 (size=5170) 2024-12-03T23:24:46,259 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39829 is added to blk_1073741844_1020 (size=5170) 2024-12-03T23:24:46,259 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45491 is added to blk_1073741844_1020 (size=5170) 2024-12-03T23:24:46,261 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=108 B at sequenceid=2 (bloomFilter=true), to=hdfs://localhost:35563/hbase/data/default/testReplayEditsWrittenIntoWAL/998b84cb1ff32c517b3d84e9f464853f/.tmp/a/c6b33ea0722a4396864d9726b7dd2969 2024-12-03T23:24:46,303 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35563/hbase/data/default/testReplayEditsWrittenIntoWAL/998b84cb1ff32c517b3d84e9f464853f/.tmp/a/c6b33ea0722a4396864d9726b7dd2969 as hdfs://localhost:35563/hbase/data/default/testReplayEditsWrittenIntoWAL/998b84cb1ff32c517b3d84e9f464853f/a/c6b33ea0722a4396864d9726b7dd2969 2024-12-03T23:24:46,313 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35563/hbase/data/default/testReplayEditsWrittenIntoWAL/998b84cb1ff32c517b3d84e9f464853f/a/c6b33ea0722a4396864d9726b7dd2969, entries=2, sequenceid=2, filesize=5.0 K 2024-12-03T23:24:46,319 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for 998b84cb1ff32c517b3d84e9f464853f in 129ms, sequenceid=2, compaction requested=false; wal=null 2024-12-03T23:24:46,321 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:35563/hbase/data/default/testReplayEditsWrittenIntoWAL/998b84cb1ff32c517b3d84e9f464853f/recovered.edits/0000000000000000002 2024-12-03T23:24:46,322 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 998b84cb1ff32c517b3d84e9f464853f 2024-12-03T23:24:46,322 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 998b84cb1ff32c517b3d84e9f464853f 2024-12-03T23:24:46,325 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 998b84cb1ff32c517b3d84e9f464853f 2024-12-03T23:24:46,329 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35563/hbase/data/default/testReplayEditsWrittenIntoWAL/998b84cb1ff32c517b3d84e9f464853f/recovered.edits/2.seqid, newMaxSeqId=2, maxSeqId=1 2024-12-03T23:24:46,330 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 998b84cb1ff32c517b3d84e9f464853f; next sequenceid=3; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64733390, jitterRate=-0.03539732098579407}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-03T23:24:46,332 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 998b84cb1ff32c517b3d84e9f464853f: Writing region info on filesystem at 1733268286166Initializing all the Stores at 1733268286168 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733268286168Obtaining lock to block concurrent updates at 1733268286188 (+20 ms)Preparing flush snapshotting stores in 998b84cb1ff32c517b3d84e9f464853f at 1733268286188Finished memstore snapshotting testReplayEditsWrittenIntoWAL,,1733268285910.998b84cb1ff32c517b3d84e9f464853f., syncing WAL and waiting on mvcc, flushsize=dataSize=108, getHeapSize=496, getOffHeapSize=0, getCellsCount=2 at 1733268286191 (+3 ms)Flushing stores of testReplayEditsWrittenIntoWAL,,1733268285910.998b84cb1ff32c517b3d84e9f464853f. at 1733268286191Flushing 998b84cb1ff32c517b3d84e9f464853f/a: creating writer at 1733268286192 (+1 ms)Flushing 998b84cb1ff32c517b3d84e9f464853f/a: appending metadata at 1733268286232 (+40 ms)Flushing 998b84cb1ff32c517b3d84e9f464853f/a: closing flushed file at 1733268286236 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6dec95d: reopening flushed file at 1733268286301 (+65 ms)Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for 998b84cb1ff32c517b3d84e9f464853f in 129ms, sequenceid=2, compaction requested=false; wal=null at 1733268286319 (+18 ms)Cleaning up temporary data from old regions at 1733268286322 (+3 ms)Region opened successfully at 1733268286332 (+10 ms) 2024-12-03T23:24:46,367 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayCompressed#testNameConflictWhenSplit0 Thread=379 (was 368) Potentially hanging thread: PacketResponder: BP-2057003906-172.17.0.2-1733268279817:blk_1073741843_1019, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-0-hdfs://localhost:35563/hbase-prefix:default java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-8-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-8-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-773788190_22 at /127.0.0.1:58292 [Waiting for operation #4] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-773788190_22 at /127.0.0.1:38024 [Receiving block BP-2057003906-172.17.0.2-1733268279817:blk_1073741843_1019] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-2057003906-172.17.0.2-1733268279817:blk_1073741843_1019, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-2057003906-172.17.0.2-1733268279817:blk_1073741843_1019, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-773788190_22 at /127.0.0.1:51196 [Receiving block BP-2057003906-172.17.0.2-1733268279817:blk_1073741843_1019] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-773788190_22 at /127.0.0.1:37890 [Waiting for operation #9] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-773788190_22 at /127.0.0.1:51016 [Waiting for operation #9] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: TestAsyncWALReplay-pool-0 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-8-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-773788190_22 at /127.0.0.1:58374 [Receiving block BP-2057003906-172.17.0.2-1733268279817:blk_1073741843_1019] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=703 (was 611) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=239 (was 239), ProcessCount=11 (was 11), AvailableMemoryMB=8180 (was 8192) 2024-12-03T23:24:46,379 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayCompressed#testNameConflictWhenSplit1 Thread=379, OpenFileDescriptor=703, MaxFileDescriptor=1048576, SystemLoadAverage=239, ProcessCount=11, AvailableMemoryMB=8180 2024-12-03T23:24:46,403 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-03T23:24:46,406 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-03T23:24:46,407 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-03T23:24:46,411 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-33381182, suffix=, logDir=hdfs://localhost:35563/hbase/WALs/hregion-33381182, archiveDir=hdfs://localhost:35563/hbase/oldWALs, maxLogs=32 2024-12-03T23:24:46,425 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-33381182/hregion-33381182.1733268286412, exclude list is [], retry=0 2024-12-03T23:24:46,430 DEBUG [AsyncFSWAL-10-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39829,DS-e6270720-ef81-458e-b5d6-959d16c4c378,DISK] 2024-12-03T23:24:46,430 DEBUG [AsyncFSWAL-10-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41487,DS-459d168f-61f7-403b-9139-91eca369c54e,DISK] 2024-12-03T23:24:46,430 DEBUG [AsyncFSWAL-10-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45491,DS-d55e5535-c0ce-4dc2-a45c-8e70eb010e56,DISK] 2024-12-03T23:24:46,434 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-33381182/hregion-33381182.1733268286412 2024-12-03T23:24:46,434 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:38065:38065),(127.0.0.1/127.0.0.1:36361:36361),(127.0.0.1/127.0.0.1:34451:34451)] 2024-12-03T23:24:46,435 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 7feb6d0f2b516be233c5e4c7b2e7dcac, NAME => 'testReplayEditsWrittenIntoWAL,,1733268286404.7feb6d0f2b516be233c5e4c7b2e7dcac.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenIntoWAL', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35563/hbase 2024-12-03T23:24:46,448 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45491 is added to blk_1073741846_1022 (size=64) 2024-12-03T23:24:46,448 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41487 is added to blk_1073741846_1022 (size=64) 2024-12-03T23:24:46,449 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39829 is added to blk_1073741846_1022 (size=64) 2024-12-03T23:24:46,450 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1733268286404.7feb6d0f2b516be233c5e4c7b2e7dcac.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T23:24:46,452 INFO [StoreOpener-7feb6d0f2b516be233c5e4c7b2e7dcac-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 7feb6d0f2b516be233c5e4c7b2e7dcac 2024-12-03T23:24:46,455 INFO [StoreOpener-7feb6d0f2b516be233c5e4c7b2e7dcac-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7feb6d0f2b516be233c5e4c7b2e7dcac columnFamilyName a 2024-12-03T23:24:46,455 DEBUG [StoreOpener-7feb6d0f2b516be233c5e4c7b2e7dcac-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T23:24:46,456 INFO [StoreOpener-7feb6d0f2b516be233c5e4c7b2e7dcac-1 {}] regionserver.HStore(327): Store=7feb6d0f2b516be233c5e4c7b2e7dcac/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T23:24:46,456 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 7feb6d0f2b516be233c5e4c7b2e7dcac 2024-12-03T23:24:46,457 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35563/hbase/data/default/testReplayEditsWrittenIntoWAL/7feb6d0f2b516be233c5e4c7b2e7dcac 2024-12-03T23:24:46,458 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35563/hbase/data/default/testReplayEditsWrittenIntoWAL/7feb6d0f2b516be233c5e4c7b2e7dcac 2024-12-03T23:24:46,458 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 7feb6d0f2b516be233c5e4c7b2e7dcac 2024-12-03T23:24:46,458 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 7feb6d0f2b516be233c5e4c7b2e7dcac 2024-12-03T23:24:46,461 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 7feb6d0f2b516be233c5e4c7b2e7dcac 2024-12-03T23:24:46,464 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35563/hbase/data/default/testReplayEditsWrittenIntoWAL/7feb6d0f2b516be233c5e4c7b2e7dcac/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T23:24:46,465 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 7feb6d0f2b516be233c5e4c7b2e7dcac; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67172354, jitterRate=9.460747241973877E-4}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-03T23:24:46,465 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 7feb6d0f2b516be233c5e4c7b2e7dcac: Writing region info on filesystem at 1733268286450Initializing all the Stores at 1733268286452 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733268286452Cleaning up temporary data from old regions at 1733268286458 (+6 ms)Region opened successfully at 1733268286465 (+7 ms) 2024-12-03T23:24:46,465 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 7feb6d0f2b516be233c5e4c7b2e7dcac, disabling compactions & flushes 2024-12-03T23:24:46,465 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1733268286404.7feb6d0f2b516be233c5e4c7b2e7dcac. 2024-12-03T23:24:46,465 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1733268286404.7feb6d0f2b516be233c5e4c7b2e7dcac. 2024-12-03T23:24:46,466 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1733268286404.7feb6d0f2b516be233c5e4c7b2e7dcac. after waiting 0 ms 2024-12-03T23:24:46,466 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1733268286404.7feb6d0f2b516be233c5e4c7b2e7dcac. 2024-12-03T23:24:46,466 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1733268286404.7feb6d0f2b516be233c5e4c7b2e7dcac. 2024-12-03T23:24:46,466 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 7feb6d0f2b516be233c5e4c7b2e7dcac: Waiting for close lock at 1733268286465Disabling compacts and flushes for region at 1733268286465Disabling writes for close at 1733268286466 (+1 ms)Writing region close event to WAL at 1733268286466Closed at 1733268286466 2024-12-03T23:24:46,470 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39829 is added to blk_1073741845_1021 (size=93) 2024-12-03T23:24:46,470 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45491 is added to blk_1073741845_1021 (size=93) 2024-12-03T23:24:46,470 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41487 is added to blk_1073741845_1021 (size=93) 2024-12-03T23:24:46,473 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-03T23:24:46,473 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-33381182:(num 1733268286412) 2024-12-03T23:24:46,474 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-12-03T23:24:46,484 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41487 is added to blk_1073741847_1023 (size=276) 2024-12-03T23:24:46,485 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45491 is added to blk_1073741847_1023 (size=276) 2024-12-03T23:24:46,485 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39829 is added to blk_1073741847_1023 (size=276) 2024-12-03T23:24:46,487 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-12-03T23:24:46,499 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45491 is added to blk_1073741848_1024 (size=230) 2024-12-03T23:24:46,500 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39829 is added to blk_1073741848_1024 (size=230) 2024-12-03T23:24:46,500 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41487 is added to blk_1073741848_1024 (size=230) 2024-12-03T23:24:46,517 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:35563/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733268286402/wal-2, size=230 (230bytes) 2024-12-03T23:24:46,518 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:35563/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733268286402/wal-2 2024-12-03T23:24:46,519 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:35563/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733268286402/wal-2 after 1ms 2024-12-03T23:24:46,522 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:35563/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733268286402/wal-2: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-03T23:24:46,523 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:35563/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733268286402/wal-2 took 6ms 2024-12-03T23:24:46,525 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:35563/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733268286402/wal-2 so closing down 2024-12-03T23:24:46,525 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-03T23:24:46,528 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000002-wal-2.temp 2024-12-03T23:24:46,529 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:35563/hbase/data/default/testReplayEditsWrittenIntoWAL/7feb6d0f2b516be233c5e4c7b2e7dcac/recovered.edits/0000000000000000002-wal-2.temp 2024-12-03T23:24:46,530 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-03T23:24:46,538 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45491 is added to blk_1073741849_1025 (size=230) 2024-12-03T23:24:46,538 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39829 is added to blk_1073741849_1025 (size=230) 2024-12-03T23:24:46,538 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41487 is added to blk_1073741849_1025 (size=230) 2024-12-03T23:24:46,539 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:35563/hbase/data/default/testReplayEditsWrittenIntoWAL/7feb6d0f2b516be233c5e4c7b2e7dcac/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-12-03T23:24:46,542 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:35563/hbase/data/default/testReplayEditsWrittenIntoWAL/7feb6d0f2b516be233c5e4c7b2e7dcac/recovered.edits/0000000000000000002-wal-2.temp to hdfs://localhost:35563/hbase/data/default/testReplayEditsWrittenIntoWAL/7feb6d0f2b516be233c5e4c7b2e7dcac/recovered.edits/0000000000000000002 2024-12-03T23:24:46,542 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 1 edits across 1 Regions in 19 ms; skipped=0; WAL=hdfs://localhost:35563/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733268286402/wal-2, size=230, length=230, corrupted=false, cancelled=false 2024-12-03T23:24:46,542 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:35563/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733268286402/wal-2, journal: Splitting hdfs://localhost:35563/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733268286402/wal-2, size=230 (230bytes) at 1733268286518Finishing writing output for hdfs://localhost:35563/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733268286402/wal-2 so closing down at 1733268286525 (+7 ms)Creating recovered edits writer path=hdfs://localhost:35563/hbase/data/default/testReplayEditsWrittenIntoWAL/7feb6d0f2b516be233c5e4c7b2e7dcac/recovered.edits/0000000000000000002-wal-2.temp at 1733268286529 (+4 ms)3 split writer threads finished at 1733268286530 (+1 ms)Closed recovered edits writer path=hdfs://localhost:35563/hbase/data/default/testReplayEditsWrittenIntoWAL/7feb6d0f2b516be233c5e4c7b2e7dcac/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1733268286539 (+9 ms)Rename recovered edits hdfs://localhost:35563/hbase/data/default/testReplayEditsWrittenIntoWAL/7feb6d0f2b516be233c5e4c7b2e7dcac/recovered.edits/0000000000000000002-wal-2.temp to hdfs://localhost:35563/hbase/data/default/testReplayEditsWrittenIntoWAL/7feb6d0f2b516be233c5e4c7b2e7dcac/recovered.edits/0000000000000000002 at 1733268286542 (+3 ms)Processed 1 edits across 1 Regions in 19 ms; skipped=0; WAL=hdfs://localhost:35563/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733268286402/wal-2, size=230, length=230, corrupted=false, cancelled=false at 1733268286542 2024-12-03T23:24:46,557 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:35563/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733268286402/wal-1, size=276 (276bytes) 2024-12-03T23:24:46,557 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:35563/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733268286402/wal-1 2024-12-03T23:24:46,558 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:35563/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733268286402/wal-1 after 1ms 2024-12-03T23:24:46,562 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:35563/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733268286402/wal-1: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-03T23:24:46,562 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:35563/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733268286402/wal-1 took 5ms 2024-12-03T23:24:46,564 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:35563/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733268286402/wal-1 so closing down 2024-12-03T23:24:46,565 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-03T23:24:46,567 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal-1.temp 2024-12-03T23:24:46,568 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:35563/hbase/data/default/testReplayEditsWrittenIntoWAL/7feb6d0f2b516be233c5e4c7b2e7dcac/recovered.edits/0000000000000000001-wal-1.temp 2024-12-03T23:24:46,568 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-03T23:24:46,576 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45491 is added to blk_1073741850_1026 (size=276) 2024-12-03T23:24:46,576 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39829 is added to blk_1073741850_1026 (size=276) 2024-12-03T23:24:46,577 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41487 is added to blk_1073741850_1026 (size=276) 2024-12-03T23:24:46,577 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:35563/hbase/data/default/testReplayEditsWrittenIntoWAL/7feb6d0f2b516be233c5e4c7b2e7dcac/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) 2024-12-03T23:24:46,581 DEBUG [split-log-closeStream-pool-0 {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:35563/hbase/data/default/testReplayEditsWrittenIntoWAL/7feb6d0f2b516be233c5e4c7b2e7dcac/recovered.edits/0000000000000000002: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-03T23:24:46,583 WARN [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(175): Found existing old edits file. It could be the result of a previous failed split attempt or we have duplicated wal entries. Deleting hdfs://localhost:35563/hbase/data/default/testReplayEditsWrittenIntoWAL/7feb6d0f2b516be233c5e4c7b2e7dcac/recovered.edits/0000000000000000002, length=230 2024-12-03T23:24:46,585 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:35563/hbase/data/default/testReplayEditsWrittenIntoWAL/7feb6d0f2b516be233c5e4c7b2e7dcac/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:35563/hbase/data/default/testReplayEditsWrittenIntoWAL/7feb6d0f2b516be233c5e4c7b2e7dcac/recovered.edits/0000000000000000002 2024-12-03T23:24:46,585 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 2 edits across 1 Regions in 23 ms; skipped=0; WAL=hdfs://localhost:35563/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733268286402/wal-1, size=276, length=276, corrupted=false, cancelled=false 2024-12-03T23:24:46,585 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:35563/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733268286402/wal-1, journal: Splitting hdfs://localhost:35563/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733268286402/wal-1, size=276 (276bytes) at 1733268286557Finishing writing output for hdfs://localhost:35563/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733268286402/wal-1 so closing down at 1733268286564 (+7 ms)Creating recovered edits writer path=hdfs://localhost:35563/hbase/data/default/testReplayEditsWrittenIntoWAL/7feb6d0f2b516be233c5e4c7b2e7dcac/recovered.edits/0000000000000000001-wal-1.temp at 1733268286568 (+4 ms)3 split writer threads finished at 1733268286569 (+1 ms)Closed recovered edits writer path=hdfs://localhost:35563/hbase/data/default/testReplayEditsWrittenIntoWAL/7feb6d0f2b516be233c5e4c7b2e7dcac/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) at 1733268286577 (+8 ms)Rename recovered edits hdfs://localhost:35563/hbase/data/default/testReplayEditsWrittenIntoWAL/7feb6d0f2b516be233c5e4c7b2e7dcac/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:35563/hbase/data/default/testReplayEditsWrittenIntoWAL/7feb6d0f2b516be233c5e4c7b2e7dcac/recovered.edits/0000000000000000002 at 1733268286585 (+8 ms)Processed 2 edits across 1 Regions in 23 ms; skipped=0; WAL=hdfs://localhost:35563/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733268286402/wal-1, size=276, length=276, corrupted=false, cancelled=false at 1733268286585 2024-12-03T23:24:46,585 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-03T23:24:46,588 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:35563/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733268286402, archiveDir=hdfs://localhost:35563/hbase/oldWALs, maxLogs=32 2024-12-03T23:24:46,601 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testnameconflictwhensplit1-manual,16010,1733268286402/wal.1733268286588, exclude list is [], retry=0 2024-12-03T23:24:46,605 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39829,DS-e6270720-ef81-458e-b5d6-959d16c4c378,DISK] 2024-12-03T23:24:46,605 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41487,DS-459d168f-61f7-403b-9139-91eca369c54e,DISK] 2024-12-03T23:24:46,605 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45491,DS-d55e5535-c0ce-4dc2-a45c-8e70eb010e56,DISK] 2024-12-03T23:24:46,608 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testnameconflictwhensplit1-manual,16010,1733268286402/wal.1733268286588 2024-12-03T23:24:46,608 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:38065:38065),(127.0.0.1/127.0.0.1:36361:36361),(127.0.0.1/127.0.0.1:34451:34451)] 2024-12-03T23:24:46,609 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 7feb6d0f2b516be233c5e4c7b2e7dcac, NAME => 'testReplayEditsWrittenIntoWAL,,1733268286404.7feb6d0f2b516be233c5e4c7b2e7dcac.', STARTKEY => '', ENDKEY => ''} 2024-12-03T23:24:46,609 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1733268286404.7feb6d0f2b516be233c5e4c7b2e7dcac.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T23:24:46,609 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 7feb6d0f2b516be233c5e4c7b2e7dcac 2024-12-03T23:24:46,609 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 7feb6d0f2b516be233c5e4c7b2e7dcac 2024-12-03T23:24:46,611 INFO [StoreOpener-7feb6d0f2b516be233c5e4c7b2e7dcac-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 7feb6d0f2b516be233c5e4c7b2e7dcac 2024-12-03T23:24:46,613 INFO [StoreOpener-7feb6d0f2b516be233c5e4c7b2e7dcac-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7feb6d0f2b516be233c5e4c7b2e7dcac columnFamilyName a 2024-12-03T23:24:46,613 DEBUG [StoreOpener-7feb6d0f2b516be233c5e4c7b2e7dcac-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T23:24:46,614 INFO [StoreOpener-7feb6d0f2b516be233c5e4c7b2e7dcac-1 {}] regionserver.HStore(327): Store=7feb6d0f2b516be233c5e4c7b2e7dcac/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T23:24:46,614 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 7feb6d0f2b516be233c5e4c7b2e7dcac 2024-12-03T23:24:46,615 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35563/hbase/data/default/testReplayEditsWrittenIntoWAL/7feb6d0f2b516be233c5e4c7b2e7dcac 2024-12-03T23:24:46,617 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:35563/hbase/data/default/testReplayEditsWrittenIntoWAL/7feb6d0f2b516be233c5e4c7b2e7dcac 2024-12-03T23:24:46,618 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:35563/hbase/data/default/testReplayEditsWrittenIntoWAL/7feb6d0f2b516be233c5e4c7b2e7dcac/recovered.edits/0000000000000000002 2024-12-03T23:24:46,621 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:35563/hbase/data/default/testReplayEditsWrittenIntoWAL/7feb6d0f2b516be233c5e4c7b2e7dcac/recovered.edits/0000000000000000002: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-03T23:24:46,623 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 2, skipped 0, firstSequenceIdInLog=1, maxSequenceIdInLog=2, path=hdfs://localhost:35563/hbase/data/default/testReplayEditsWrittenIntoWAL/7feb6d0f2b516be233c5e4c7b2e7dcac/recovered.edits/0000000000000000002 2024-12-03T23:24:46,623 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 7feb6d0f2b516be233c5e4c7b2e7dcac 1/1 column families, dataSize=108 B heapSize=512 B 2024-12-03T23:24:46,639 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35563/hbase/data/default/testReplayEditsWrittenIntoWAL/7feb6d0f2b516be233c5e4c7b2e7dcac/.tmp/a/b40313c28bc74db581d9ae9a2b6a059a is 58, key is testReplayEditsWrittenIntoWAL/a:1/1733268286473/Put/seqid=0 2024-12-03T23:24:46,649 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41487 is added to blk_1073741852_1028 (size=5170) 2024-12-03T23:24:46,649 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45491 is added to blk_1073741852_1028 (size=5170) 2024-12-03T23:24:46,649 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39829 is added to blk_1073741852_1028 (size=5170) 2024-12-03T23:24:46,650 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=108 B at sequenceid=2 (bloomFilter=true), to=hdfs://localhost:35563/hbase/data/default/testReplayEditsWrittenIntoWAL/7feb6d0f2b516be233c5e4c7b2e7dcac/.tmp/a/b40313c28bc74db581d9ae9a2b6a059a 2024-12-03T23:24:46,660 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35563/hbase/data/default/testReplayEditsWrittenIntoWAL/7feb6d0f2b516be233c5e4c7b2e7dcac/.tmp/a/b40313c28bc74db581d9ae9a2b6a059a as hdfs://localhost:35563/hbase/data/default/testReplayEditsWrittenIntoWAL/7feb6d0f2b516be233c5e4c7b2e7dcac/a/b40313c28bc74db581d9ae9a2b6a059a 2024-12-03T23:24:46,669 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35563/hbase/data/default/testReplayEditsWrittenIntoWAL/7feb6d0f2b516be233c5e4c7b2e7dcac/a/b40313c28bc74db581d9ae9a2b6a059a, entries=2, sequenceid=2, filesize=5.0 K 2024-12-03T23:24:46,670 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for 7feb6d0f2b516be233c5e4c7b2e7dcac in 47ms, sequenceid=2, compaction requested=false; wal=null 2024-12-03T23:24:46,671 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:35563/hbase/data/default/testReplayEditsWrittenIntoWAL/7feb6d0f2b516be233c5e4c7b2e7dcac/recovered.edits/0000000000000000002 2024-12-03T23:24:46,671 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 7feb6d0f2b516be233c5e4c7b2e7dcac 2024-12-03T23:24:46,671 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 7feb6d0f2b516be233c5e4c7b2e7dcac 2024-12-03T23:24:46,674 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 7feb6d0f2b516be233c5e4c7b2e7dcac 2024-12-03T23:24:46,677 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35563/hbase/data/default/testReplayEditsWrittenIntoWAL/7feb6d0f2b516be233c5e4c7b2e7dcac/recovered.edits/2.seqid, newMaxSeqId=2, maxSeqId=1 2024-12-03T23:24:46,678 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 7feb6d0f2b516be233c5e4c7b2e7dcac; next sequenceid=3; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66135749, jitterRate=-0.014500543475151062}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-03T23:24:46,679 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 7feb6d0f2b516be233c5e4c7b2e7dcac: Writing region info on filesystem at 1733268286609Initializing all the Stores at 1733268286611 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733268286611Obtaining lock to block concurrent updates at 1733268286623 (+12 ms)Preparing flush snapshotting stores in 7feb6d0f2b516be233c5e4c7b2e7dcac at 1733268286623Finished memstore snapshotting testReplayEditsWrittenIntoWAL,,1733268286404.7feb6d0f2b516be233c5e4c7b2e7dcac., syncing WAL and waiting on mvcc, flushsize=dataSize=108, getHeapSize=496, getOffHeapSize=0, getCellsCount=2 at 1733268286623Flushing stores of testReplayEditsWrittenIntoWAL,,1733268286404.7feb6d0f2b516be233c5e4c7b2e7dcac. at 1733268286623Flushing 7feb6d0f2b516be233c5e4c7b2e7dcac/a: creating writer at 1733268286623Flushing 7feb6d0f2b516be233c5e4c7b2e7dcac/a: appending metadata at 1733268286638 (+15 ms)Flushing 7feb6d0f2b516be233c5e4c7b2e7dcac/a: closing flushed file at 1733268286638Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2534be9b: reopening flushed file at 1733268286659 (+21 ms)Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for 7feb6d0f2b516be233c5e4c7b2e7dcac in 47ms, sequenceid=2, compaction requested=false; wal=null at 1733268286670 (+11 ms)Cleaning up temporary data from old regions at 1733268286671 (+1 ms)Region opened successfully at 1733268286679 (+8 ms) 2024-12-03T23:24:46,696 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayCompressed#testNameConflictWhenSplit1 Thread=389 (was 379) Potentially hanging thread: PacketResponder: BP-2057003906-172.17.0.2-1733268279817:blk_1073741851_1027, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-2057003906-172.17.0.2-1733268279817:blk_1073741851_1027, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-773788190_22 at /127.0.0.1:58292 [Waiting for operation #8] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-2057003906-172.17.0.2-1733268279817:blk_1073741851_1027, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-10-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-773788190_22 at /127.0.0.1:37890 [Waiting for operation #13] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-10-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-773788190_22 at /127.0.0.1:51016 [Waiting for operation #16] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-10-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-773788190_22 at /127.0.0.1:58438 [Receiving block BP-2057003906-172.17.0.2-1733268279817:blk_1073741851_1027] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-773788190_22 at /127.0.0.1:38090 [Receiving block BP-2057003906-172.17.0.2-1733268279817:blk_1073741851_1027] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-773788190_22 at /127.0.0.1:51244 [Receiving block BP-2057003906-172.17.0.2-1733268279817:blk_1073741851_1027] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=785 (was 703) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=239 (was 239), ProcessCount=11 (was 11), AvailableMemoryMB=8182 (was 8180) - AvailableMemoryMB LEAK? - 2024-12-03T23:24:46,708 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayCompressed#testReplayEditsWrittenIntoWAL Thread=389, OpenFileDescriptor=785, MaxFileDescriptor=1048576, SystemLoadAverage=239, ProcessCount=11, AvailableMemoryMB=8182 2024-12-03T23:24:46,723 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-03T23:24:46,725 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-03T23:24:46,726 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-03T23:24:46,729 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-44375259, suffix=, logDir=hdfs://localhost:35563/hbase/WALs/hregion-44375259, archiveDir=hdfs://localhost:35563/hbase/oldWALs, maxLogs=32 2024-12-03T23:24:46,743 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-44375259/hregion-44375259.1733268286730, exclude list is [], retry=0 2024-12-03T23:24:46,747 DEBUG [AsyncFSWAL-12-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41487,DS-459d168f-61f7-403b-9139-91eca369c54e,DISK] 2024-12-03T23:24:46,747 DEBUG [AsyncFSWAL-12-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39829,DS-e6270720-ef81-458e-b5d6-959d16c4c378,DISK] 2024-12-03T23:24:46,748 DEBUG [AsyncFSWAL-12-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45491,DS-d55e5535-c0ce-4dc2-a45c-8e70eb010e56,DISK] 2024-12-03T23:24:46,751 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-44375259/hregion-44375259.1733268286730 2024-12-03T23:24:46,751 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36361:36361),(127.0.0.1/127.0.0.1:38065:38065),(127.0.0.1/127.0.0.1:34451:34451)] 2024-12-03T23:24:46,752 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 15204530ecd4d36824d69618a641d312, NAME => 'testReplayEditsWrittenIntoWAL,,1733268286724.15204530ecd4d36824d69618a641d312.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenIntoWAL', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35563/hbase 2024-12-03T23:24:46,764 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39829 is added to blk_1073741854_1030 (size=64) 2024-12-03T23:24:46,765 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41487 is added to blk_1073741854_1030 (size=64) 2024-12-03T23:24:46,765 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45491 is added to blk_1073741854_1030 (size=64) 2024-12-03T23:24:46,766 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1733268286724.15204530ecd4d36824d69618a641d312.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T23:24:46,769 INFO [StoreOpener-15204530ecd4d36824d69618a641d312-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 15204530ecd4d36824d69618a641d312 2024-12-03T23:24:46,770 INFO [StoreOpener-15204530ecd4d36824d69618a641d312-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 15204530ecd4d36824d69618a641d312 columnFamilyName a 2024-12-03T23:24:46,770 DEBUG [StoreOpener-15204530ecd4d36824d69618a641d312-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T23:24:46,771 INFO [StoreOpener-15204530ecd4d36824d69618a641d312-1 {}] regionserver.HStore(327): Store=15204530ecd4d36824d69618a641d312/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T23:24:46,771 INFO [StoreOpener-15204530ecd4d36824d69618a641d312-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 15204530ecd4d36824d69618a641d312 2024-12-03T23:24:46,773 INFO [StoreOpener-15204530ecd4d36824d69618a641d312-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 15204530ecd4d36824d69618a641d312 columnFamilyName b 2024-12-03T23:24:46,773 DEBUG [StoreOpener-15204530ecd4d36824d69618a641d312-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T23:24:46,773 INFO [StoreOpener-15204530ecd4d36824d69618a641d312-1 {}] regionserver.HStore(327): Store=15204530ecd4d36824d69618a641d312/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T23:24:46,773 INFO [StoreOpener-15204530ecd4d36824d69618a641d312-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 15204530ecd4d36824d69618a641d312 2024-12-03T23:24:46,775 INFO [StoreOpener-15204530ecd4d36824d69618a641d312-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 15204530ecd4d36824d69618a641d312 columnFamilyName c 2024-12-03T23:24:46,775 DEBUG [StoreOpener-15204530ecd4d36824d69618a641d312-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T23:24:46,776 INFO [StoreOpener-15204530ecd4d36824d69618a641d312-1 {}] regionserver.HStore(327): Store=15204530ecd4d36824d69618a641d312/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T23:24:46,776 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 15204530ecd4d36824d69618a641d312 2024-12-03T23:24:46,777 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35563/hbase/data/default/testReplayEditsWrittenIntoWAL/15204530ecd4d36824d69618a641d312 2024-12-03T23:24:46,777 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35563/hbase/data/default/testReplayEditsWrittenIntoWAL/15204530ecd4d36824d69618a641d312 2024-12-03T23:24:46,779 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 15204530ecd4d36824d69618a641d312 2024-12-03T23:24:46,779 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 15204530ecd4d36824d69618a641d312 2024-12-03T23:24:46,779 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenIntoWAL descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-03T23:24:46,781 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 15204530ecd4d36824d69618a641d312 2024-12-03T23:24:46,783 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35563/hbase/data/default/testReplayEditsWrittenIntoWAL/15204530ecd4d36824d69618a641d312/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T23:24:46,784 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 15204530ecd4d36824d69618a641d312; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60906618, jitterRate=-0.09242066740989685}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-03T23:24:46,784 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 15204530ecd4d36824d69618a641d312: Writing region info on filesystem at 1733268286767Initializing all the Stores at 1733268286768 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733268286768Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733268286768Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733268286768Cleaning up temporary data from old regions at 1733268286779 (+11 ms)Region opened successfully at 1733268286784 (+5 ms) 2024-12-03T23:24:46,784 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 15204530ecd4d36824d69618a641d312, disabling compactions & flushes 2024-12-03T23:24:46,784 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1733268286724.15204530ecd4d36824d69618a641d312. 2024-12-03T23:24:46,784 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1733268286724.15204530ecd4d36824d69618a641d312. 2024-12-03T23:24:46,784 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1733268286724.15204530ecd4d36824d69618a641d312. after waiting 0 ms 2024-12-03T23:24:46,785 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1733268286724.15204530ecd4d36824d69618a641d312. 2024-12-03T23:24:46,785 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1733268286724.15204530ecd4d36824d69618a641d312. 2024-12-03T23:24:46,785 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 15204530ecd4d36824d69618a641d312: Waiting for close lock at 1733268286784Disabling compacts and flushes for region at 1733268286784Disabling writes for close at 1733268286784Writing region close event to WAL at 1733268286785 (+1 ms)Closed at 1733268286785 2024-12-03T23:24:46,789 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39829 is added to blk_1073741853_1029 (size=93) 2024-12-03T23:24:46,789 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45491 is added to blk_1073741853_1029 (size=93) 2024-12-03T23:24:46,789 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41487 is added to blk_1073741853_1029 (size=93) 2024-12-03T23:24:46,792 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-03T23:24:46,792 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-44375259:(num 1733268286730) 2024-12-03T23:24:46,792 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-03T23:24:46,794 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:35563/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733268286722, archiveDir=hdfs://localhost:35563/hbase/oldWALs, maxLogs=32 2024-12-03T23:24:46,808 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733268286722/wal.1733268286795, exclude list is [], retry=0 2024-12-03T23:24:46,811 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45491,DS-d55e5535-c0ce-4dc2-a45c-8e70eb010e56,DISK] 2024-12-03T23:24:46,812 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39829,DS-e6270720-ef81-458e-b5d6-959d16c4c378,DISK] 2024-12-03T23:24:46,812 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41487,DS-459d168f-61f7-403b-9139-91eca369c54e,DISK] 2024-12-03T23:24:46,814 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733268286722/wal.1733268286795 2024-12-03T23:24:46,816 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34451:34451),(127.0.0.1/127.0.0.1:38065:38065),(127.0.0.1/127.0.0.1:36361:36361)] 2024-12-03T23:24:47,067 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:35563/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733268286722/wal.1733268286795, size=0 (0bytes) 2024-12-03T23:24:47,067 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:35563/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733268286722/wal.1733268286795 might be still open, length is 0 2024-12-03T23:24:47,067 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:35563/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733268286722/wal.1733268286795 2024-12-03T23:24:47,068 WARN [IPC Server handler 1 on default port 35563 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733268286722/wal.1733268286795 has not been closed. Lease recovery is in progress. RecoveryId = 1032 for block blk_1073741855_1031 2024-12-03T23:24:47,069 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:35563/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733268286722/wal.1733268286795 after 2ms 2024-12-03T23:24:47,786 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-773788190_22 at /127.0.0.1:38128 [Receiving block BP-2057003906-172.17.0.2-1733268279817:blk_1073741855_1031] {}] datanode.DataXceiver(331): 127.0.0.1:45491:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:38128 dst: /127.0.0.1:45491 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:45491 remote=/127.0.0.1:38128]. Total timeout mills is 60000, 59246 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T23:24:47,787 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-773788190_22 at /127.0.0.1:58476 [Receiving block BP-2057003906-172.17.0.2-1733268279817:blk_1073741855_1031] {}] datanode.DataXceiver(331): 127.0.0.1:39829:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:58476 dst: /127.0.0.1:39829 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T23:24:47,787 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-773788190_22 at /127.0.0.1:51284 [Receiving block BP-2057003906-172.17.0.2-1733268279817:blk_1073741855_1031] {}] datanode.DataXceiver(331): 127.0.0.1:41487:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:51284 dst: /127.0.0.1:41487 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T23:24:47,794 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45491 is added to blk_1073741855_1032 (size=200589) 2024-12-03T23:24:47,797 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39829 is added to blk_1073741855_1032 (size=200589) 2024-12-03T23:24:47,798 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41487 is added to blk_1073741855_1032 (size=200589) 2024-12-03T23:24:50,983 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-03T23:24:51,030 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-12-03T23:24:51,070 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:35563/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733268286722/wal.1733268286795 after 4003ms 2024-12-03T23:24:51,074 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:35563/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733268286722/wal.1733268286795: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-03T23:24:51,076 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:35563/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733268286722/wal.1733268286795 took 4009ms 2024-12-03T23:24:51,080 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal.1733268286795.temp 2024-12-03T23:24:51,083 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:35563/hbase/data/default/testReplayEditsWrittenIntoWAL/15204530ecd4d36824d69618a641d312/recovered.edits/0000000000000000001-wal.1733268286795.temp 2024-12-03T23:24:51,196 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:35563/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733268286722/wal.1733268286795; continuing. 2024-12-03T23:24:51,197 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:35563/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733268286722/wal.1733268286795 so closing down 2024-12-03T23:24:51,197 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-03T23:24:51,197 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-03T23:24:51,200 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45491 is added to blk_1073741856_1033 (size=200597) 2024-12-03T23:24:51,201 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41487 is added to blk_1073741856_1033 (size=200597) 2024-12-03T23:24:51,202 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39829 is added to blk_1073741856_1033 (size=200597) 2024-12-03T23:24:51,202 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:35563/hbase/data/default/testReplayEditsWrittenIntoWAL/15204530ecd4d36824d69618a641d312/recovered.edits/0000000000000000001-wal.1733268286795.temp (wrote 3002 edits, skipped 0 edits in 42 ms) 2024-12-03T23:24:51,204 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:35563/hbase/data/default/testReplayEditsWrittenIntoWAL/15204530ecd4d36824d69618a641d312/recovered.edits/0000000000000000001-wal.1733268286795.temp to hdfs://localhost:35563/hbase/data/default/testReplayEditsWrittenIntoWAL/15204530ecd4d36824d69618a641d312/recovered.edits/0000000000000003002 2024-12-03T23:24:51,204 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 3002 edits across 1 Regions in 128 ms; skipped=0; WAL=hdfs://localhost:35563/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733268286722/wal.1733268286795, size=0, length=0, corrupted=false, cancelled=false 2024-12-03T23:24:51,204 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:35563/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733268286722/wal.1733268286795, journal: Splitting hdfs://localhost:35563/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733268286722/wal.1733268286795, size=0 (0bytes) at 1733268287067Creating recovered edits writer path=hdfs://localhost:35563/hbase/data/default/testReplayEditsWrittenIntoWAL/15204530ecd4d36824d69618a641d312/recovered.edits/0000000000000000001-wal.1733268286795.temp at 1733268291083 (+4016 ms)Split 1024 edits, skipped 0 edits. at 1733268291142 (+59 ms)Split 2048 edits, skipped 0 edits. at 1733268291169 (+27 ms)Finishing writing output for hdfs://localhost:35563/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733268286722/wal.1733268286795 so closing down at 1733268291197 (+28 ms)3 split writer threads finished at 1733268291197Closed recovered edits writer path=hdfs://localhost:35563/hbase/data/default/testReplayEditsWrittenIntoWAL/15204530ecd4d36824d69618a641d312/recovered.edits/0000000000000000001-wal.1733268286795.temp (wrote 3002 edits, skipped 0 edits in 42 ms) at 1733268291202 (+5 ms)Rename recovered edits hdfs://localhost:35563/hbase/data/default/testReplayEditsWrittenIntoWAL/15204530ecd4d36824d69618a641d312/recovered.edits/0000000000000000001-wal.1733268286795.temp to hdfs://localhost:35563/hbase/data/default/testReplayEditsWrittenIntoWAL/15204530ecd4d36824d69618a641d312/recovered.edits/0000000000000003002 at 1733268291204 (+2 ms)Processed 3002 edits across 1 Regions in 128 ms; skipped=0; WAL=hdfs://localhost:35563/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733268286722/wal.1733268286795, size=0, length=0, corrupted=false, cancelled=false at 1733268291204 2024-12-03T23:24:51,207 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:35563/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733268286722/wal.1733268286795 to hdfs://localhost:35563/hbase/oldWALs/wal.1733268286795 2024-12-03T23:24:51,208 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:35563/hbase/data/default/testReplayEditsWrittenIntoWAL/15204530ecd4d36824d69618a641d312/recovered.edits/0000000000000003002 2024-12-03T23:24:51,208 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-03T23:24:51,211 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:35563/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733268286722, archiveDir=hdfs://localhost:35563/hbase/oldWALs, maxLogs=32 2024-12-03T23:24:51,224 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733268286722/wal.1733268291211, exclude list is [], retry=0 2024-12-03T23:24:51,227 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41487,DS-459d168f-61f7-403b-9139-91eca369c54e,DISK] 2024-12-03T23:24:51,228 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45491,DS-d55e5535-c0ce-4dc2-a45c-8e70eb010e56,DISK] 2024-12-03T23:24:51,228 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39829,DS-e6270720-ef81-458e-b5d6-959d16c4c378,DISK] 2024-12-03T23:24:51,230 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733268286722/wal.1733268291211 2024-12-03T23:24:51,232 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36361:36361),(127.0.0.1/127.0.0.1:34451:34451),(127.0.0.1/127.0.0.1:38065:38065)] 2024-12-03T23:24:51,232 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1733268286724.15204530ecd4d36824d69618a641d312.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T23:24:51,234 INFO [StoreOpener-15204530ecd4d36824d69618a641d312-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 15204530ecd4d36824d69618a641d312 2024-12-03T23:24:51,235 INFO [StoreOpener-15204530ecd4d36824d69618a641d312-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 15204530ecd4d36824d69618a641d312 columnFamilyName a 2024-12-03T23:24:51,235 DEBUG [StoreOpener-15204530ecd4d36824d69618a641d312-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T23:24:51,236 INFO [StoreOpener-15204530ecd4d36824d69618a641d312-1 {}] regionserver.HStore(327): Store=15204530ecd4d36824d69618a641d312/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T23:24:51,236 INFO [StoreOpener-15204530ecd4d36824d69618a641d312-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 15204530ecd4d36824d69618a641d312 2024-12-03T23:24:51,237 INFO [StoreOpener-15204530ecd4d36824d69618a641d312-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 15204530ecd4d36824d69618a641d312 columnFamilyName b 2024-12-03T23:24:51,237 DEBUG [StoreOpener-15204530ecd4d36824d69618a641d312-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T23:24:51,238 INFO [StoreOpener-15204530ecd4d36824d69618a641d312-1 {}] regionserver.HStore(327): Store=15204530ecd4d36824d69618a641d312/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T23:24:51,238 INFO [StoreOpener-15204530ecd4d36824d69618a641d312-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 15204530ecd4d36824d69618a641d312 2024-12-03T23:24:51,239 INFO [StoreOpener-15204530ecd4d36824d69618a641d312-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 15204530ecd4d36824d69618a641d312 columnFamilyName c 2024-12-03T23:24:51,239 DEBUG [StoreOpener-15204530ecd4d36824d69618a641d312-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T23:24:51,239 INFO [StoreOpener-15204530ecd4d36824d69618a641d312-1 {}] regionserver.HStore(327): Store=15204530ecd4d36824d69618a641d312/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T23:24:51,239 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 15204530ecd4d36824d69618a641d312 2024-12-03T23:24:51,240 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35563/hbase/data/default/testReplayEditsWrittenIntoWAL/15204530ecd4d36824d69618a641d312 2024-12-03T23:24:51,242 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:35563/hbase/data/default/testReplayEditsWrittenIntoWAL/15204530ecd4d36824d69618a641d312 2024-12-03T23:24:51,243 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:35563/hbase/data/default/testReplayEditsWrittenIntoWAL/15204530ecd4d36824d69618a641d312/recovered.edits/0000000000000003002 2024-12-03T23:24:51,246 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:35563/hbase/data/default/testReplayEditsWrittenIntoWAL/15204530ecd4d36824d69618a641d312/recovered.edits/0000000000000003002: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-03T23:24:51,278 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-12-03T23:24:51,648 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 15204530ecd4d36824d69618a641d312 3/3 column families, dataSize=42.49 KB heapSize=100.11 KB 2024-12-03T23:24:51,689 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35563/hbase/data/default/testReplayEditsWrittenIntoWAL/15204530ecd4d36824d69618a641d312/.tmp/a/c70fbc9fdd5d4340a5e4643f700e0c3f is 62, key is testReplayEditsWrittenIntoWAL/a:100/1733268286828/Put/seqid=0 2024-12-03T23:24:51,698 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45491 is added to blk_1073741858_1035 (size=50463) 2024-12-03T23:24:51,698 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39829 is added to blk_1073741858_1035 (size=50463) 2024-12-03T23:24:51,699 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41487 is added to blk_1073741858_1035 (size=50463) 2024-12-03T23:24:51,700 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=754 (bloomFilter=true), to=hdfs://localhost:35563/hbase/data/default/testReplayEditsWrittenIntoWAL/15204530ecd4d36824d69618a641d312/.tmp/a/c70fbc9fdd5d4340a5e4643f700e0c3f 2024-12-03T23:24:51,709 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35563/hbase/data/default/testReplayEditsWrittenIntoWAL/15204530ecd4d36824d69618a641d312/.tmp/a/c70fbc9fdd5d4340a5e4643f700e0c3f as hdfs://localhost:35563/hbase/data/default/testReplayEditsWrittenIntoWAL/15204530ecd4d36824d69618a641d312/a/c70fbc9fdd5d4340a5e4643f700e0c3f 2024-12-03T23:24:51,716 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35563/hbase/data/default/testReplayEditsWrittenIntoWAL/15204530ecd4d36824d69618a641d312/a/c70fbc9fdd5d4340a5e4643f700e0c3f, entries=754, sequenceid=754, filesize=49.3 K 2024-12-03T23:24:51,717 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~42.49 KB/43512, heapSize ~99.59 KB/101984, currentSize=0 B/0 for 15204530ecd4d36824d69618a641d312 in 69ms, sequenceid=754, compaction requested=false; wal=null 2024-12-03T23:24:51,747 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-12-03T23:24:51,748 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 15204530ecd4d36824d69618a641d312 3/3 column families, dataSize=42.49 KB heapSize=100.11 KB 2024-12-03T23:24:51,758 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35563/hbase/data/default/testReplayEditsWrittenIntoWAL/15204530ecd4d36824d69618a641d312/.tmp/a/cfe07f0a386748d08a4facfae7321144 is 62, key is testReplayEditsWrittenIntoWAL/a:754/1733268286893/Put/seqid=0 2024-12-03T23:24:51,771 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39829 is added to blk_1073741859_1036 (size=20072) 2024-12-03T23:24:51,771 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41487 is added to blk_1073741859_1036 (size=20072) 2024-12-03T23:24:51,772 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45491 is added to blk_1073741859_1036 (size=20072) 2024-12-03T23:24:51,773 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.93 KB at sequenceid=1508 (bloomFilter=true), to=hdfs://localhost:35563/hbase/data/default/testReplayEditsWrittenIntoWAL/15204530ecd4d36824d69618a641d312/.tmp/a/cfe07f0a386748d08a4facfae7321144 2024-12-03T23:24:51,809 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35563/hbase/data/default/testReplayEditsWrittenIntoWAL/15204530ecd4d36824d69618a641d312/.tmp/b/b5d7c32d7bc248bfae5745f135b395f6 is 62, key is testReplayEditsWrittenIntoWAL/b:100/1733268286924/Put/seqid=0 2024-12-03T23:24:51,818 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45491 is added to blk_1073741860_1037 (size=35835) 2024-12-03T23:24:51,819 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41487 is added to blk_1073741860_1037 (size=35835) 2024-12-03T23:24:51,819 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39829 is added to blk_1073741860_1037 (size=35835) 2024-12-03T23:24:51,820 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=28.56 KB at sequenceid=1508 (bloomFilter=true), to=hdfs://localhost:35563/hbase/data/default/testReplayEditsWrittenIntoWAL/15204530ecd4d36824d69618a641d312/.tmp/b/b5d7c32d7bc248bfae5745f135b395f6 2024-12-03T23:24:51,830 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35563/hbase/data/default/testReplayEditsWrittenIntoWAL/15204530ecd4d36824d69618a641d312/.tmp/a/cfe07f0a386748d08a4facfae7321144 as hdfs://localhost:35563/hbase/data/default/testReplayEditsWrittenIntoWAL/15204530ecd4d36824d69618a641d312/a/cfe07f0a386748d08a4facfae7321144 2024-12-03T23:24:51,838 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35563/hbase/data/default/testReplayEditsWrittenIntoWAL/15204530ecd4d36824d69618a641d312/a/cfe07f0a386748d08a4facfae7321144, entries=246, sequenceid=1508, filesize=19.6 K 2024-12-03T23:24:51,840 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35563/hbase/data/default/testReplayEditsWrittenIntoWAL/15204530ecd4d36824d69618a641d312/.tmp/b/b5d7c32d7bc248bfae5745f135b395f6 as hdfs://localhost:35563/hbase/data/default/testReplayEditsWrittenIntoWAL/15204530ecd4d36824d69618a641d312/b/b5d7c32d7bc248bfae5745f135b395f6 2024-12-03T23:24:51,847 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35563/hbase/data/default/testReplayEditsWrittenIntoWAL/15204530ecd4d36824d69618a641d312/b/b5d7c32d7bc248bfae5745f135b395f6, entries=508, sequenceid=1508, filesize=35.0 K 2024-12-03T23:24:51,847 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~42.49 KB/43512, heapSize ~99.83 KB/102224, currentSize=0 B/0 for 15204530ecd4d36824d69618a641d312 in 99ms, sequenceid=1508, compaction requested=false; wal=null 2024-12-03T23:24:51,862 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-12-03T23:24:51,863 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 15204530ecd4d36824d69618a641d312 3/3 column families, dataSize=42.49 KB heapSize=100.11 KB 2024-12-03T23:24:51,870 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35563/hbase/data/default/testReplayEditsWrittenIntoWAL/15204530ecd4d36824d69618a641d312/.tmp/b/e0c63d4f232745ec86c427d49650c018 is 62, key is testReplayEditsWrittenIntoWAL/b:508/1733268286945/Put/seqid=0 2024-12-03T23:24:51,879 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45491 is added to blk_1073741861_1038 (size=35082) 2024-12-03T23:24:51,879 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41487 is added to blk_1073741861_1038 (size=35082) 2024-12-03T23:24:51,880 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39829 is added to blk_1073741861_1038 (size=35082) 2024-12-03T23:24:51,880 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=27.87 KB at sequenceid=2262 (bloomFilter=true), to=hdfs://localhost:35563/hbase/data/default/testReplayEditsWrittenIntoWAL/15204530ecd4d36824d69618a641d312/.tmp/b/e0c63d4f232745ec86c427d49650c018 2024-12-03T23:24:51,911 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35563/hbase/data/default/testReplayEditsWrittenIntoWAL/15204530ecd4d36824d69618a641d312/.tmp/c/6e5964d9ecf94998931934c39f25f4f6 is 62, key is testReplayEditsWrittenIntoWAL/c:100/1733268286979/Put/seqid=0 2024-12-03T23:24:51,922 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41487 is added to blk_1073741862_1039 (size=20825) 2024-12-03T23:24:51,922 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45491 is added to blk_1073741862_1039 (size=20825) 2024-12-03T23:24:51,923 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39829 is added to blk_1073741862_1039 (size=20825) 2024-12-03T23:24:51,923 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=14.63 KB at sequenceid=2262 (bloomFilter=true), to=hdfs://localhost:35563/hbase/data/default/testReplayEditsWrittenIntoWAL/15204530ecd4d36824d69618a641d312/.tmp/c/6e5964d9ecf94998931934c39f25f4f6 2024-12-03T23:24:51,931 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35563/hbase/data/default/testReplayEditsWrittenIntoWAL/15204530ecd4d36824d69618a641d312/.tmp/b/e0c63d4f232745ec86c427d49650c018 as hdfs://localhost:35563/hbase/data/default/testReplayEditsWrittenIntoWAL/15204530ecd4d36824d69618a641d312/b/e0c63d4f232745ec86c427d49650c018 2024-12-03T23:24:51,938 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35563/hbase/data/default/testReplayEditsWrittenIntoWAL/15204530ecd4d36824d69618a641d312/b/e0c63d4f232745ec86c427d49650c018, entries=492, sequenceid=2262, filesize=34.3 K 2024-12-03T23:24:51,939 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35563/hbase/data/default/testReplayEditsWrittenIntoWAL/15204530ecd4d36824d69618a641d312/.tmp/c/6e5964d9ecf94998931934c39f25f4f6 as hdfs://localhost:35563/hbase/data/default/testReplayEditsWrittenIntoWAL/15204530ecd4d36824d69618a641d312/c/6e5964d9ecf94998931934c39f25f4f6 2024-12-03T23:24:51,946 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35563/hbase/data/default/testReplayEditsWrittenIntoWAL/15204530ecd4d36824d69618a641d312/c/6e5964d9ecf94998931934c39f25f4f6, entries=262, sequenceid=2262, filesize=20.3 K 2024-12-03T23:24:51,946 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~42.49 KB/43512, heapSize ~99.83 KB/102224, currentSize=0 B/0 for 15204530ecd4d36824d69618a641d312 in 84ms, sequenceid=2262, compaction requested=false; wal=null 2024-12-03T23:24:51,956 WARN [Time-limited test {}] regionserver.HRegion(5722): No family for cell testReplayEditsWrittenIntoWAL/another family:testReplayEditsWrittenIntoWAL/1733268287030/Put/vlen=29/seqid=0 in region testReplayEditsWrittenIntoWAL,,1733268286724.15204530ecd4d36824d69618a641d312. 2024-12-03T23:24:51,959 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 3001, skipped 1, firstSequenceIdInLog=1, maxSequenceIdInLog=3002, path=hdfs://localhost:35563/hbase/data/default/testReplayEditsWrittenIntoWAL/15204530ecd4d36824d69618a641d312/recovered.edits/0000000000000003002 2024-12-03T23:24:51,959 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-12-03T23:24:51,959 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 15204530ecd4d36824d69618a641d312 3/3 column families, dataSize=41.85 KB heapSize=98.89 KB 2024-12-03T23:24:51,969 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35563/hbase/data/default/testReplayEditsWrittenIntoWAL/15204530ecd4d36824d69618a641d312/.tmp/c/cdd4bdcd7cf54e45a49b58c211e0547d is 62, key is testReplayEditsWrittenIntoWAL/c:262/1733268286988/Put/seqid=0 2024-12-03T23:24:51,984 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41487 is added to blk_1073741863_1040 (size=50301) 2024-12-03T23:24:51,984 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45491 is added to blk_1073741863_1040 (size=50301) 2024-12-03T23:24:51,985 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39829 is added to blk_1073741863_1040 (size=50301) 2024-12-03T23:24:51,985 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=41.85 KB at sequenceid=3002 (bloomFilter=true), to=hdfs://localhost:35563/hbase/data/default/testReplayEditsWrittenIntoWAL/15204530ecd4d36824d69618a641d312/.tmp/c/cdd4bdcd7cf54e45a49b58c211e0547d 2024-12-03T23:24:51,993 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for cdd4bdcd7cf54e45a49b58c211e0547d 2024-12-03T23:24:51,995 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35563/hbase/data/default/testReplayEditsWrittenIntoWAL/15204530ecd4d36824d69618a641d312/.tmp/c/cdd4bdcd7cf54e45a49b58c211e0547d as hdfs://localhost:35563/hbase/data/default/testReplayEditsWrittenIntoWAL/15204530ecd4d36824d69618a641d312/c/cdd4bdcd7cf54e45a49b58c211e0547d 2024-12-03T23:24:52,002 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for cdd4bdcd7cf54e45a49b58c211e0547d 2024-12-03T23:24:52,002 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35563/hbase/data/default/testReplayEditsWrittenIntoWAL/15204530ecd4d36824d69618a641d312/c/cdd4bdcd7cf54e45a49b58c211e0547d, entries=739, sequenceid=3002, filesize=49.1 K 2024-12-03T23:24:52,002 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~41.85 KB/42854, heapSize ~98.38 KB/100736, currentSize=0 B/0 for 15204530ecd4d36824d69618a641d312 in 43ms, sequenceid=3002, compaction requested=false; wal=null 2024-12-03T23:24:52,003 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:35563/hbase/data/default/testReplayEditsWrittenIntoWAL/15204530ecd4d36824d69618a641d312/recovered.edits/0000000000000003002 2024-12-03T23:24:52,005 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 15204530ecd4d36824d69618a641d312 2024-12-03T23:24:52,005 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 15204530ecd4d36824d69618a641d312 2024-12-03T23:24:52,005 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenIntoWAL descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-03T23:24:52,007 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 15204530ecd4d36824d69618a641d312 2024-12-03T23:24:52,010 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35563/hbase/data/default/testReplayEditsWrittenIntoWAL/15204530ecd4d36824d69618a641d312/recovered.edits/3002.seqid, newMaxSeqId=3002, maxSeqId=1 2024-12-03T23:24:52,011 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 15204530ecd4d36824d69618a641d312; next sequenceid=3003; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=204800, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69109330, jitterRate=0.029809266328811646}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-03T23:24:52,011 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 15204530ecd4d36824d69618a641d312: Writing region info on filesystem at 1733268291232Initializing all the Stores at 1733268291233 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733268291233Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733268291234 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733268291234Cleaning up temporary data from old regions at 1733268292005 (+771 ms)Region opened successfully at 1733268292011 (+6 ms) 2024-12-03T23:24:52,058 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 15204530ecd4d36824d69618a641d312, disabling compactions & flushes 2024-12-03T23:24:52,058 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1733268286724.15204530ecd4d36824d69618a641d312. 2024-12-03T23:24:52,058 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1733268286724.15204530ecd4d36824d69618a641d312. 2024-12-03T23:24:52,058 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1733268286724.15204530ecd4d36824d69618a641d312. after waiting 0 ms 2024-12-03T23:24:52,058 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1733268286724.15204530ecd4d36824d69618a641d312. 2024-12-03T23:24:52,060 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1733268286724.15204530ecd4d36824d69618a641d312. 2024-12-03T23:24:52,060 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 15204530ecd4d36824d69618a641d312: Waiting for close lock at 1733268292058Disabling compacts and flushes for region at 1733268292058Disabling writes for close at 1733268292058Writing region close event to WAL at 1733268292060 (+2 ms)Closed at 1733268292060 2024-12-03T23:24:52,064 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39829 is added to blk_1073741857_1034 (size=93) 2024-12-03T23:24:52,064 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41487 is added to blk_1073741857_1034 (size=93) 2024-12-03T23:24:52,065 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45491 is added to blk_1073741857_1034 (size=93) 2024-12-03T23:24:52,068 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-03T23:24:52,068 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1733268291211) 2024-12-03T23:24:52,089 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayCompressed#testReplayEditsWrittenIntoWAL Thread=405 (was 389) Potentially hanging thread: LeaseRenewer:jenkins.replay.wal.secondtime@localhost:35563 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1987708253) connection to localhost/127.0.0.1:41371 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1323030589_22 at /127.0.0.1:43784 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1323030589_22 at /127.0.0.1:52060 [Waiting for operation #23] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1987708253) connection to localhost/127.0.0.1:35563 from jenkins.replay.wal.secondtime java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:40347 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/246ab1eb-e73c-fcd7-a82a-bcf4c34f0add/cluster_eb757c39-dfd9-3b39-78fd-eea66f4cff1d/data/data1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1987708253) connection to localhost/127.0.0.1:40347 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1323030589_22 at /127.0.0.1:40620 [Waiting for operation #12] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-12-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:41371 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/246ab1eb-e73c-fcd7-a82a-bcf4c34f0add/cluster_eb757c39-dfd9-3b39-78fd-eea66f4cff1d/data/data4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-12-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: AsyncFSWAL-12-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1323030589_22 at /127.0.0.1:40606 [Waiting for operation #6] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/246ab1eb-e73c-fcd7-a82a-bcf4c34f0add/cluster_eb757c39-dfd9-3b39-78fd-eea66f4cff1d/data/data2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/246ab1eb-e73c-fcd7-a82a-bcf4c34f0add/cluster_eb757c39-dfd9-3b39-78fd-eea66f4cff1d/data/data3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=865 (was 785) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=228 (was 239), ProcessCount=11 (was 11), AvailableMemoryMB=8110 (was 8182) 2024-12-03T23:24:52,102 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayCompressed#test2727 Thread=405, OpenFileDescriptor=865, MaxFileDescriptor=1048576, SystemLoadAverage=228, ProcessCount=11, AvailableMemoryMB=8108 2024-12-03T23:24:52,118 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-03T23:24:52,120 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-03T23:24:52,121 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-03T23:24:52,124 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-75544671, suffix=, logDir=hdfs://localhost:35563/hbase/WALs/hregion-75544671, archiveDir=hdfs://localhost:35563/hbase/oldWALs, maxLogs=32 2024-12-03T23:24:52,137 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-75544671/hregion-75544671.1733268292125, exclude list is [], retry=0 2024-12-03T23:24:52,141 DEBUG [AsyncFSWAL-14-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45491,DS-d55e5535-c0ce-4dc2-a45c-8e70eb010e56,DISK] 2024-12-03T23:24:52,141 DEBUG [AsyncFSWAL-14-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39829,DS-e6270720-ef81-458e-b5d6-959d16c4c378,DISK] 2024-12-03T23:24:52,141 DEBUG [AsyncFSWAL-14-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41487,DS-459d168f-61f7-403b-9139-91eca369c54e,DISK] 2024-12-03T23:24:52,143 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-75544671/hregion-75544671.1733268292125 2024-12-03T23:24:52,144 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34451:34451),(127.0.0.1/127.0.0.1:38065:38065),(127.0.0.1/127.0.0.1:36361:36361)] 2024-12-03T23:24:52,144 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 055c1016111bbf69b0c28b72b5285e19, NAME => 'test2727,,1733268292119.055c1016111bbf69b0c28b72b5285e19.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='test2727', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35563/hbase 2024-12-03T23:24:52,152 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39829 is added to blk_1073741865_1042 (size=43) 2024-12-03T23:24:52,153 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41487 is added to blk_1073741865_1042 (size=43) 2024-12-03T23:24:52,153 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45491 is added to blk_1073741865_1042 (size=43) 2024-12-03T23:24:52,155 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated test2727,,1733268292119.055c1016111bbf69b0c28b72b5285e19.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T23:24:52,156 INFO [StoreOpener-055c1016111bbf69b0c28b72b5285e19-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 055c1016111bbf69b0c28b72b5285e19 2024-12-03T23:24:52,158 INFO [StoreOpener-055c1016111bbf69b0c28b72b5285e19-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 055c1016111bbf69b0c28b72b5285e19 columnFamilyName a 2024-12-03T23:24:52,158 DEBUG [StoreOpener-055c1016111bbf69b0c28b72b5285e19-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T23:24:52,158 INFO [StoreOpener-055c1016111bbf69b0c28b72b5285e19-1 {}] regionserver.HStore(327): Store=055c1016111bbf69b0c28b72b5285e19/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T23:24:52,159 INFO [StoreOpener-055c1016111bbf69b0c28b72b5285e19-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 055c1016111bbf69b0c28b72b5285e19 2024-12-03T23:24:52,160 INFO [StoreOpener-055c1016111bbf69b0c28b72b5285e19-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 055c1016111bbf69b0c28b72b5285e19 columnFamilyName b 2024-12-03T23:24:52,160 DEBUG [StoreOpener-055c1016111bbf69b0c28b72b5285e19-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T23:24:52,161 INFO [StoreOpener-055c1016111bbf69b0c28b72b5285e19-1 {}] regionserver.HStore(327): Store=055c1016111bbf69b0c28b72b5285e19/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T23:24:52,161 INFO [StoreOpener-055c1016111bbf69b0c28b72b5285e19-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 055c1016111bbf69b0c28b72b5285e19 2024-12-03T23:24:52,162 INFO [StoreOpener-055c1016111bbf69b0c28b72b5285e19-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 055c1016111bbf69b0c28b72b5285e19 columnFamilyName c 2024-12-03T23:24:52,163 DEBUG [StoreOpener-055c1016111bbf69b0c28b72b5285e19-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T23:24:52,163 INFO [StoreOpener-055c1016111bbf69b0c28b72b5285e19-1 {}] regionserver.HStore(327): Store=055c1016111bbf69b0c28b72b5285e19/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T23:24:52,163 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 055c1016111bbf69b0c28b72b5285e19 2024-12-03T23:24:52,164 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35563/hbase/data/default/test2727/055c1016111bbf69b0c28b72b5285e19 2024-12-03T23:24:52,164 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35563/hbase/data/default/test2727/055c1016111bbf69b0c28b72b5285e19 2024-12-03T23:24:52,166 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 055c1016111bbf69b0c28b72b5285e19 2024-12-03T23:24:52,166 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 055c1016111bbf69b0c28b72b5285e19 2024-12-03T23:24:52,166 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table test2727 descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-03T23:24:52,168 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 055c1016111bbf69b0c28b72b5285e19 2024-12-03T23:24:52,171 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35563/hbase/data/default/test2727/055c1016111bbf69b0c28b72b5285e19/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T23:24:52,172 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 055c1016111bbf69b0c28b72b5285e19; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59219207, jitterRate=-0.11756505072116852}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-03T23:24:52,173 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 055c1016111bbf69b0c28b72b5285e19: Writing region info on filesystem at 1733268292155Initializing all the Stores at 1733268292156 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733268292156Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733268292156Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733268292156Cleaning up temporary data from old regions at 1733268292166 (+10 ms)Region opened successfully at 1733268292173 (+7 ms) 2024-12-03T23:24:52,173 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 055c1016111bbf69b0c28b72b5285e19, disabling compactions & flushes 2024-12-03T23:24:52,173 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region test2727,,1733268292119.055c1016111bbf69b0c28b72b5285e19. 2024-12-03T23:24:52,173 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on test2727,,1733268292119.055c1016111bbf69b0c28b72b5285e19. 2024-12-03T23:24:52,173 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on test2727,,1733268292119.055c1016111bbf69b0c28b72b5285e19. after waiting 0 ms 2024-12-03T23:24:52,173 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region test2727,,1733268292119.055c1016111bbf69b0c28b72b5285e19. 2024-12-03T23:24:52,174 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed test2727,,1733268292119.055c1016111bbf69b0c28b72b5285e19. 2024-12-03T23:24:52,174 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 055c1016111bbf69b0c28b72b5285e19: Waiting for close lock at 1733268292173Disabling compacts and flushes for region at 1733268292173Disabling writes for close at 1733268292173Writing region close event to WAL at 1733268292173Closed at 1733268292173 2024-12-03T23:24:52,177 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45491 is added to blk_1073741864_1041 (size=93) 2024-12-03T23:24:52,177 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41487 is added to blk_1073741864_1041 (size=93) 2024-12-03T23:24:52,178 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39829 is added to blk_1073741864_1041 (size=93) 2024-12-03T23:24:52,181 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-03T23:24:52,181 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-75544671:(num 1733268292125) 2024-12-03T23:24:52,181 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-03T23:24:52,183 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:35563/hbase/WALs/test2727-manual,16010,1733268292118, archiveDir=hdfs://localhost:35563/hbase/oldWALs, maxLogs=32 2024-12-03T23:24:52,204 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/test2727-manual,16010,1733268292118/wal.1733268292184, exclude list is [], retry=0 2024-12-03T23:24:52,207 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41487,DS-459d168f-61f7-403b-9139-91eca369c54e,DISK] 2024-12-03T23:24:52,208 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39829,DS-e6270720-ef81-458e-b5d6-959d16c4c378,DISK] 2024-12-03T23:24:52,208 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45491,DS-d55e5535-c0ce-4dc2-a45c-8e70eb010e56,DISK] 2024-12-03T23:24:52,211 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/test2727-manual,16010,1733268292118/wal.1733268292184 2024-12-03T23:24:52,211 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36361:36361),(127.0.0.1/127.0.0.1:38065:38065),(127.0.0.1/127.0.0.1:34451:34451)] 2024-12-03T23:24:52,385 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45491 is added to blk_1073741866_1043 (size=200357) 2024-12-03T23:24:52,385 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41487 is added to blk_1073741866_1043 (size=200357) 2024-12-03T23:24:52,385 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39829 is added to blk_1073741866_1043 (size=200357) 2024-12-03T23:24:52,386 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /hbase/WALs/test2727-manual,16010,1733268292118/wal.1733268292184 not finished, retry = 0 2024-12-03T23:24:52,505 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:35563/hbase/WALs/test2727-manual,16010,1733268292118/wal.1733268292184, size=195.7 K (200357bytes) 2024-12-03T23:24:52,505 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:35563/hbase/WALs/test2727-manual,16010,1733268292118/wal.1733268292184 2024-12-03T23:24:52,505 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:35563/hbase/WALs/test2727-manual,16010,1733268292118/wal.1733268292184 after 0ms 2024-12-03T23:24:52,509 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:35563/hbase/WALs/test2727-manual,16010,1733268292118/wal.1733268292184: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-03T23:24:52,511 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:35563/hbase/WALs/test2727-manual,16010,1733268292118/wal.1733268292184 took 7ms 2024-12-03T23:24:52,516 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal.1733268292184.temp 2024-12-03T23:24:52,518 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:35563/hbase/data/default/test2727/055c1016111bbf69b0c28b72b5285e19/recovered.edits/0000000000000000001-wal.1733268292184.temp 2024-12-03T23:24:52,580 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:35563/hbase/WALs/test2727-manual,16010,1733268292118/wal.1733268292184 so closing down 2024-12-03T23:24:52,580 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-03T23:24:52,580 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-03T23:24:52,583 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45491 is added to blk_1073741867_1044 (size=200357) 2024-12-03T23:24:52,583 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41487 is added to blk_1073741867_1044 (size=200357) 2024-12-03T23:24:52,584 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39829 is added to blk_1073741867_1044 (size=200357) 2024-12-03T23:24:52,585 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:35563/hbase/data/default/test2727/055c1016111bbf69b0c28b72b5285e19/recovered.edits/0000000000000000001-wal.1733268292184.temp (wrote 3000 edits, skipped 0 edits in 21 ms) 2024-12-03T23:24:52,587 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:35563/hbase/data/default/test2727/055c1016111bbf69b0c28b72b5285e19/recovered.edits/0000000000000000001-wal.1733268292184.temp to hdfs://localhost:35563/hbase/data/default/test2727/055c1016111bbf69b0c28b72b5285e19/recovered.edits/0000000000000003000 2024-12-03T23:24:52,587 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 3000 edits across 1 Regions in 75 ms; skipped=0; WAL=hdfs://localhost:35563/hbase/WALs/test2727-manual,16010,1733268292118/wal.1733268292184, size=195.7 K, length=200357, corrupted=false, cancelled=false 2024-12-03T23:24:52,587 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:35563/hbase/WALs/test2727-manual,16010,1733268292118/wal.1733268292184, journal: Splitting hdfs://localhost:35563/hbase/WALs/test2727-manual,16010,1733268292118/wal.1733268292184, size=195.7 K (200357bytes) at 1733268292505Creating recovered edits writer path=hdfs://localhost:35563/hbase/data/default/test2727/055c1016111bbf69b0c28b72b5285e19/recovered.edits/0000000000000000001-wal.1733268292184.temp at 1733268292518 (+13 ms)Split 1024 edits, skipped 0 edits. at 1733268292538 (+20 ms)Split 2048 edits, skipped 0 edits. at 1733268292560 (+22 ms)Finishing writing output for hdfs://localhost:35563/hbase/WALs/test2727-manual,16010,1733268292118/wal.1733268292184 so closing down at 1733268292580 (+20 ms)3 split writer threads finished at 1733268292580Closed recovered edits writer path=hdfs://localhost:35563/hbase/data/default/test2727/055c1016111bbf69b0c28b72b5285e19/recovered.edits/0000000000000000001-wal.1733268292184.temp (wrote 3000 edits, skipped 0 edits in 21 ms) at 1733268292585 (+5 ms)Rename recovered edits hdfs://localhost:35563/hbase/data/default/test2727/055c1016111bbf69b0c28b72b5285e19/recovered.edits/0000000000000000001-wal.1733268292184.temp to hdfs://localhost:35563/hbase/data/default/test2727/055c1016111bbf69b0c28b72b5285e19/recovered.edits/0000000000000003000 at 1733268292587 (+2 ms)Processed 3000 edits across 1 Regions in 75 ms; skipped=0; WAL=hdfs://localhost:35563/hbase/WALs/test2727-manual,16010,1733268292118/wal.1733268292184, size=195.7 K, length=200357, corrupted=false, cancelled=false at 1733268292587 2024-12-03T23:24:52,589 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:35563/hbase/WALs/test2727-manual,16010,1733268292118/wal.1733268292184 to hdfs://localhost:35563/hbase/oldWALs/wal.1733268292184 2024-12-03T23:24:52,590 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:35563/hbase/data/default/test2727/055c1016111bbf69b0c28b72b5285e19/recovered.edits/0000000000000003000 2024-12-03T23:24:52,591 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-03T23:24:52,593 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:35563/hbase/WALs/test2727-manual,16010,1733268292118, archiveDir=hdfs://localhost:35563/hbase/oldWALs, maxLogs=32 2024-12-03T23:24:52,644 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/test2727-manual,16010,1733268292118/wal.1733268292622, exclude list is [], retry=0 2024-12-03T23:24:52,651 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41487,DS-459d168f-61f7-403b-9139-91eca369c54e,DISK] 2024-12-03T23:24:52,651 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39829,DS-e6270720-ef81-458e-b5d6-959d16c4c378,DISK] 2024-12-03T23:24:52,651 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45491,DS-d55e5535-c0ce-4dc2-a45c-8e70eb010e56,DISK] 2024-12-03T23:24:52,660 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/test2727-manual,16010,1733268292118/wal.1733268292622 2024-12-03T23:24:52,660 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36361:36361),(127.0.0.1/127.0.0.1:38065:38065),(127.0.0.1/127.0.0.1:34451:34451)] 2024-12-03T23:24:52,843 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45491 is added to blk_1073741868_1045 (size=200484) 2024-12-03T23:24:52,843 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41487 is added to blk_1073741868_1045 (size=200484) 2024-12-03T23:24:52,844 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39829 is added to blk_1073741868_1045 (size=200484) 2024-12-03T23:24:52,869 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:35563/hbase/WALs/test2727-manual,16010,1733268292118/wal.1733268292622, size=195.8 K (200484bytes) 2024-12-03T23:24:52,869 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:35563/hbase/WALs/test2727-manual,16010,1733268292118/wal.1733268292622 2024-12-03T23:24:52,869 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:35563/hbase/WALs/test2727-manual,16010,1733268292118/wal.1733268292622 after 0ms 2024-12-03T23:24:52,872 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:35563/hbase/WALs/test2727-manual,16010,1733268292118/wal.1733268292622: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-03T23:24:52,874 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:35563/hbase/WALs/test2727-manual,16010,1733268292118/wal.1733268292622 took 6ms 2024-12-03T23:24:52,878 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000003001-wal.1733268292622.temp 2024-12-03T23:24:52,880 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:35563/hbase/data/default/test2727/055c1016111bbf69b0c28b72b5285e19/recovered.edits/0000000000000003001-wal.1733268292622.temp 2024-12-03T23:24:52,916 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:35563/hbase/WALs/test2727-manual,16010,1733268292118/wal.1733268292622 so closing down 2024-12-03T23:24:52,917 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-03T23:24:52,917 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-03T23:24:52,919 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39829 is added to blk_1073741869_1046 (size=200484) 2024-12-03T23:24:52,920 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45491 is added to blk_1073741869_1046 (size=200484) 2024-12-03T23:24:52,920 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41487 is added to blk_1073741869_1046 (size=200484) 2024-12-03T23:24:52,921 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:35563/hbase/data/default/test2727/055c1016111bbf69b0c28b72b5285e19/recovered.edits/0000000000000003001-wal.1733268292622.temp (wrote 3000 edits, skipped 0 edits in 16 ms) 2024-12-03T23:24:52,923 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:35563/hbase/data/default/test2727/055c1016111bbf69b0c28b72b5285e19/recovered.edits/0000000000000003001-wal.1733268292622.temp to hdfs://localhost:35563/hbase/data/default/test2727/055c1016111bbf69b0c28b72b5285e19/recovered.edits/0000000000000006000 2024-12-03T23:24:52,923 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 3000 edits across 1 Regions in 49 ms; skipped=0; WAL=hdfs://localhost:35563/hbase/WALs/test2727-manual,16010,1733268292118/wal.1733268292622, size=195.8 K, length=200484, corrupted=false, cancelled=false 2024-12-03T23:24:52,923 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:35563/hbase/WALs/test2727-manual,16010,1733268292118/wal.1733268292622, journal: Splitting hdfs://localhost:35563/hbase/WALs/test2727-manual,16010,1733268292118/wal.1733268292622, size=195.8 K (200484bytes) at 1733268292869Creating recovered edits writer path=hdfs://localhost:35563/hbase/data/default/test2727/055c1016111bbf69b0c28b72b5285e19/recovered.edits/0000000000000003001-wal.1733268292622.temp at 1733268292880 (+11 ms)Split 1024 edits, skipped 0 edits. at 1733268292889 (+9 ms)Split 2048 edits, skipped 0 edits. at 1733268292903 (+14 ms)Finishing writing output for hdfs://localhost:35563/hbase/WALs/test2727-manual,16010,1733268292118/wal.1733268292622 so closing down at 1733268292916 (+13 ms)3 split writer threads finished at 1733268292917 (+1 ms)Closed recovered edits writer path=hdfs://localhost:35563/hbase/data/default/test2727/055c1016111bbf69b0c28b72b5285e19/recovered.edits/0000000000000003001-wal.1733268292622.temp (wrote 3000 edits, skipped 0 edits in 16 ms) at 1733268292921 (+4 ms)Rename recovered edits hdfs://localhost:35563/hbase/data/default/test2727/055c1016111bbf69b0c28b72b5285e19/recovered.edits/0000000000000003001-wal.1733268292622.temp to hdfs://localhost:35563/hbase/data/default/test2727/055c1016111bbf69b0c28b72b5285e19/recovered.edits/0000000000000006000 at 1733268292923 (+2 ms)Processed 3000 edits across 1 Regions in 49 ms; skipped=0; WAL=hdfs://localhost:35563/hbase/WALs/test2727-manual,16010,1733268292118/wal.1733268292622, size=195.8 K, length=200484, corrupted=false, cancelled=false at 1733268292923 2024-12-03T23:24:52,925 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:35563/hbase/WALs/test2727-manual,16010,1733268292118/wal.1733268292622 to hdfs://localhost:35563/hbase/oldWALs/wal.1733268292622 2024-12-03T23:24:52,926 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:35563/hbase/data/default/test2727/055c1016111bbf69b0c28b72b5285e19/recovered.edits/0000000000000006000 2024-12-03T23:24:52,926 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-03T23:24:52,929 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:35563/hbase/WALs/test2727-manual,16010,1733268292118, archiveDir=hdfs://localhost:35563/hbase/oldWALs, maxLogs=32 2024-12-03T23:24:52,944 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/test2727-manual,16010,1733268292118/wal.1733268292929, exclude list is [], retry=0 2024-12-03T23:24:52,947 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45491,DS-d55e5535-c0ce-4dc2-a45c-8e70eb010e56,DISK] 2024-12-03T23:24:52,948 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39829,DS-e6270720-ef81-458e-b5d6-959d16c4c378,DISK] 2024-12-03T23:24:52,948 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41487,DS-459d168f-61f7-403b-9139-91eca369c54e,DISK] 2024-12-03T23:24:52,951 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/test2727-manual,16010,1733268292118/wal.1733268292929 2024-12-03T23:24:52,951 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34451:34451),(127.0.0.1/127.0.0.1:38065:38065),(127.0.0.1/127.0.0.1:36361:36361)] 2024-12-03T23:24:52,951 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 055c1016111bbf69b0c28b72b5285e19, NAME => 'test2727,,1733268292119.055c1016111bbf69b0c28b72b5285e19.', STARTKEY => '', ENDKEY => ''} 2024-12-03T23:24:52,952 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated test2727,,1733268292119.055c1016111bbf69b0c28b72b5285e19.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T23:24:52,952 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 055c1016111bbf69b0c28b72b5285e19 2024-12-03T23:24:52,952 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 055c1016111bbf69b0c28b72b5285e19 2024-12-03T23:24:52,954 INFO [StoreOpener-055c1016111bbf69b0c28b72b5285e19-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 055c1016111bbf69b0c28b72b5285e19 2024-12-03T23:24:52,955 INFO [StoreOpener-055c1016111bbf69b0c28b72b5285e19-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 055c1016111bbf69b0c28b72b5285e19 columnFamilyName a 2024-12-03T23:24:52,955 DEBUG [StoreOpener-055c1016111bbf69b0c28b72b5285e19-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T23:24:52,956 INFO [StoreOpener-055c1016111bbf69b0c28b72b5285e19-1 {}] regionserver.HStore(327): Store=055c1016111bbf69b0c28b72b5285e19/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T23:24:52,956 INFO [StoreOpener-055c1016111bbf69b0c28b72b5285e19-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 055c1016111bbf69b0c28b72b5285e19 2024-12-03T23:24:52,957 INFO [StoreOpener-055c1016111bbf69b0c28b72b5285e19-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 055c1016111bbf69b0c28b72b5285e19 columnFamilyName b 2024-12-03T23:24:52,957 DEBUG [StoreOpener-055c1016111bbf69b0c28b72b5285e19-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T23:24:52,958 INFO [StoreOpener-055c1016111bbf69b0c28b72b5285e19-1 {}] regionserver.HStore(327): Store=055c1016111bbf69b0c28b72b5285e19/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T23:24:52,958 INFO [StoreOpener-055c1016111bbf69b0c28b72b5285e19-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 055c1016111bbf69b0c28b72b5285e19 2024-12-03T23:24:52,959 INFO [StoreOpener-055c1016111bbf69b0c28b72b5285e19-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 055c1016111bbf69b0c28b72b5285e19 columnFamilyName c 2024-12-03T23:24:52,959 DEBUG [StoreOpener-055c1016111bbf69b0c28b72b5285e19-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T23:24:52,960 INFO [StoreOpener-055c1016111bbf69b0c28b72b5285e19-1 {}] regionserver.HStore(327): Store=055c1016111bbf69b0c28b72b5285e19/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T23:24:52,960 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 055c1016111bbf69b0c28b72b5285e19 2024-12-03T23:24:52,961 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35563/hbase/data/default/test2727/055c1016111bbf69b0c28b72b5285e19 2024-12-03T23:24:52,963 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 2 recovered edits file(s) under hdfs://localhost:35563/hbase/data/default/test2727/055c1016111bbf69b0c28b72b5285e19 2024-12-03T23:24:52,964 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:35563/hbase/data/default/test2727/055c1016111bbf69b0c28b72b5285e19/recovered.edits/0000000000000003000 2024-12-03T23:24:52,967 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:35563/hbase/data/default/test2727/055c1016111bbf69b0c28b72b5285e19/recovered.edits/0000000000000003000: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-03T23:24:53,002 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 3000, skipped 0, firstSequenceIdInLog=1, maxSequenceIdInLog=3000, path=hdfs://localhost:35563/hbase/data/default/test2727/055c1016111bbf69b0c28b72b5285e19/recovered.edits/0000000000000003000 2024-12-03T23:24:53,003 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:35563/hbase/data/default/test2727/055c1016111bbf69b0c28b72b5285e19/recovered.edits/0000000000000006000 2024-12-03T23:24:53,006 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:35563/hbase/data/default/test2727/055c1016111bbf69b0c28b72b5285e19/recovered.edits/0000000000000006000: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-03T23:24:53,040 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 3000, skipped 0, firstSequenceIdInLog=3001, maxSequenceIdInLog=6000, path=hdfs://localhost:35563/hbase/data/default/test2727/055c1016111bbf69b0c28b72b5285e19/recovered.edits/0000000000000006000 2024-12-03T23:24:53,041 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 055c1016111bbf69b0c28b72b5285e19 3/3 column families, dataSize=215.51 KB heapSize=657 KB 2024-12-03T23:24:53,062 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35563/hbase/data/default/test2727/055c1016111bbf69b0c28b72b5285e19/.tmp/a/4ad3694d8a3f41c3933882a5e47cf226 is 41, key is test2727/a:100/1733268292669/Put/seqid=0 2024-12-03T23:24:53,068 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39829 is added to blk_1073741871_1048 (size=84227) 2024-12-03T23:24:53,069 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41487 is added to blk_1073741871_1048 (size=84227) 2024-12-03T23:24:53,069 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45491 is added to blk_1073741871_1048 (size=84227) 2024-12-03T23:24:53,070 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=71.84 KB at sequenceid=6000 (bloomFilter=true), to=hdfs://localhost:35563/hbase/data/default/test2727/055c1016111bbf69b0c28b72b5285e19/.tmp/a/4ad3694d8a3f41c3933882a5e47cf226 2024-12-03T23:24:53,098 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35563/hbase/data/default/test2727/055c1016111bbf69b0c28b72b5285e19/.tmp/b/0bb84302901d4ae9a5d681c0c81e4d93 is 41, key is test2727/b:100/1733268292724/Put/seqid=0 2024-12-03T23:24:53,102 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39829 is added to blk_1073741872_1049 (size=84609) 2024-12-03T23:24:53,103 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41487 is added to blk_1073741872_1049 (size=84609) 2024-12-03T23:24:53,103 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45491 is added to blk_1073741872_1049 (size=84609) 2024-12-03T23:24:53,104 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=71.84 KB at sequenceid=6000 (bloomFilter=true), to=hdfs://localhost:35563/hbase/data/default/test2727/055c1016111bbf69b0c28b72b5285e19/.tmp/b/0bb84302901d4ae9a5d681c0c81e4d93 2024-12-03T23:24:53,128 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35563/hbase/data/default/test2727/055c1016111bbf69b0c28b72b5285e19/.tmp/c/bc5c9079b4dc4a4c966d4b462c58fdaa is 41, key is test2727/c:100/1733268292785/Put/seqid=0 2024-12-03T23:24:53,136 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45491 is added to blk_1073741873_1050 (size=84609) 2024-12-03T23:24:53,136 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39829 is added to blk_1073741873_1050 (size=84609) 2024-12-03T23:24:53,136 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41487 is added to blk_1073741873_1050 (size=84609) 2024-12-03T23:24:53,258 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-12-03T23:24:53,259 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-12-03T23:24:53,260 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-03T23:24:53,260 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-12-03T23:24:53,260 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-12-03T23:24:53,260 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-12-03T23:24:53,261 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenIntoWAL 2024-12-03T23:24:53,261 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenIntoWAL Metrics about Tables on a single HBase RegionServer 2024-12-03T23:24:53,536 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=71.84 KB at sequenceid=6000 (bloomFilter=true), to=hdfs://localhost:35563/hbase/data/default/test2727/055c1016111bbf69b0c28b72b5285e19/.tmp/c/bc5c9079b4dc4a4c966d4b462c58fdaa 2024-12-03T23:24:53,544 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35563/hbase/data/default/test2727/055c1016111bbf69b0c28b72b5285e19/.tmp/a/4ad3694d8a3f41c3933882a5e47cf226 as hdfs://localhost:35563/hbase/data/default/test2727/055c1016111bbf69b0c28b72b5285e19/a/4ad3694d8a3f41c3933882a5e47cf226 2024-12-03T23:24:53,549 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35563/hbase/data/default/test2727/055c1016111bbf69b0c28b72b5285e19/a/4ad3694d8a3f41c3933882a5e47cf226, entries=2000, sequenceid=6000, filesize=82.3 K 2024-12-03T23:24:53,550 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35563/hbase/data/default/test2727/055c1016111bbf69b0c28b72b5285e19/.tmp/b/0bb84302901d4ae9a5d681c0c81e4d93 as hdfs://localhost:35563/hbase/data/default/test2727/055c1016111bbf69b0c28b72b5285e19/b/0bb84302901d4ae9a5d681c0c81e4d93 2024-12-03T23:24:53,556 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35563/hbase/data/default/test2727/055c1016111bbf69b0c28b72b5285e19/b/0bb84302901d4ae9a5d681c0c81e4d93, entries=2000, sequenceid=6000, filesize=82.6 K 2024-12-03T23:24:53,557 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35563/hbase/data/default/test2727/055c1016111bbf69b0c28b72b5285e19/.tmp/c/bc5c9079b4dc4a4c966d4b462c58fdaa as hdfs://localhost:35563/hbase/data/default/test2727/055c1016111bbf69b0c28b72b5285e19/c/bc5c9079b4dc4a4c966d4b462c58fdaa 2024-12-03T23:24:53,562 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35563/hbase/data/default/test2727/055c1016111bbf69b0c28b72b5285e19/c/bc5c9079b4dc4a4c966d4b462c58fdaa, entries=2000, sequenceid=6000, filesize=82.6 K 2024-12-03T23:24:53,562 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~215.51 KB/220680, heapSize ~656.95 KB/672720, currentSize=0 B/0 for 055c1016111bbf69b0c28b72b5285e19 in 522ms, sequenceid=6000, compaction requested=false; wal=null 2024-12-03T23:24:53,563 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:35563/hbase/data/default/test2727/055c1016111bbf69b0c28b72b5285e19/recovered.edits/0000000000000003000 2024-12-03T23:24:53,564 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:35563/hbase/data/default/test2727/055c1016111bbf69b0c28b72b5285e19/recovered.edits/0000000000000006000 2024-12-03T23:24:53,565 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 055c1016111bbf69b0c28b72b5285e19 2024-12-03T23:24:53,565 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 055c1016111bbf69b0c28b72b5285e19 2024-12-03T23:24:53,565 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table test2727 descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-03T23:24:53,567 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 055c1016111bbf69b0c28b72b5285e19 2024-12-03T23:24:53,569 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35563/hbase/data/default/test2727/055c1016111bbf69b0c28b72b5285e19/recovered.edits/6000.seqid, newMaxSeqId=6000, maxSeqId=1 2024-12-03T23:24:53,570 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 055c1016111bbf69b0c28b72b5285e19; next sequenceid=6001; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65595439, jitterRate=-0.02255178987979889}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-03T23:24:53,571 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 055c1016111bbf69b0c28b72b5285e19: Writing region info on filesystem at 1733268292952Initializing all the Stores at 1733268292953 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733268292953Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733268292954 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733268292954Obtaining lock to block concurrent updates at 1733268293041 (+87 ms)Preparing flush snapshotting stores in 055c1016111bbf69b0c28b72b5285e19 at 1733268293041Finished memstore snapshotting test2727,,1733268292119.055c1016111bbf69b0c28b72b5285e19., syncing WAL and waiting on mvcc, flushsize=dataSize=220680, getHeapSize=672720, getOffHeapSize=0, getCellsCount=6000 at 1733268293041Flushing stores of test2727,,1733268292119.055c1016111bbf69b0c28b72b5285e19. at 1733268293041Flushing 055c1016111bbf69b0c28b72b5285e19/a: creating writer at 1733268293041Flushing 055c1016111bbf69b0c28b72b5285e19/a: appending metadata at 1733268293061 (+20 ms)Flushing 055c1016111bbf69b0c28b72b5285e19/a: closing flushed file at 1733268293061Flushing 055c1016111bbf69b0c28b72b5285e19/b: creating writer at 1733268293076 (+15 ms)Flushing 055c1016111bbf69b0c28b72b5285e19/b: appending metadata at 1733268293096 (+20 ms)Flushing 055c1016111bbf69b0c28b72b5285e19/b: closing flushed file at 1733268293096Flushing 055c1016111bbf69b0c28b72b5285e19/c: creating writer at 1733268293110 (+14 ms)Flushing 055c1016111bbf69b0c28b72b5285e19/c: appending metadata at 1733268293127 (+17 ms)Flushing 055c1016111bbf69b0c28b72b5285e19/c: closing flushed file at 1733268293127Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7d4ee31f: reopening flushed file at 1733268293543 (+416 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@f131142: reopening flushed file at 1733268293549 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@33381c2a: reopening flushed file at 1733268293556 (+7 ms)Finished flush of dataSize ~215.51 KB/220680, heapSize ~656.95 KB/672720, currentSize=0 B/0 for 055c1016111bbf69b0c28b72b5285e19 in 522ms, sequenceid=6000, compaction requested=false; wal=null at 1733268293562 (+6 ms)Cleaning up temporary data from old regions at 1733268293565 (+3 ms)Region opened successfully at 1733268293571 (+6 ms) 2024-12-03T23:24:53,572 DEBUG [Time-limited test {}] wal.AbstractTestWALReplay(320): region.getOpenSeqNum(): 6001, wal3.id: 0 2024-12-03T23:24:53,572 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 055c1016111bbf69b0c28b72b5285e19, disabling compactions & flushes 2024-12-03T23:24:53,572 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region test2727,,1733268292119.055c1016111bbf69b0c28b72b5285e19. 2024-12-03T23:24:53,572 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on test2727,,1733268292119.055c1016111bbf69b0c28b72b5285e19. 2024-12-03T23:24:53,572 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on test2727,,1733268292119.055c1016111bbf69b0c28b72b5285e19. after waiting 0 ms 2024-12-03T23:24:53,572 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region test2727,,1733268292119.055c1016111bbf69b0c28b72b5285e19. 2024-12-03T23:24:53,573 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed test2727,,1733268292119.055c1016111bbf69b0c28b72b5285e19. 2024-12-03T23:24:53,573 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 055c1016111bbf69b0c28b72b5285e19: Waiting for close lock at 1733268293572Disabling compacts and flushes for region at 1733268293572Disabling writes for close at 1733268293572Writing region close event to WAL at 1733268293573 (+1 ms)Closed at 1733268293573 2024-12-03T23:24:53,577 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /hbase/WALs/test2727-manual,16010,1733268292118/wal.1733268292929 not finished, retry = 0 2024-12-03T23:24:53,577 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45491 is added to blk_1073741870_1047 (size=93) 2024-12-03T23:24:53,578 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39829 is added to blk_1073741870_1047 (size=93) 2024-12-03T23:24:53,578 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41487 is added to blk_1073741870_1047 (size=93) 2024-12-03T23:24:53,680 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-03T23:24:53,680 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1733268292929) 2024-12-03T23:24:53,694 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayCompressed#test2727 Thread=407 (was 405) Potentially hanging thread: AsyncFSWAL-14-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-773788190_22 at /127.0.0.1:43784 [Waiting for operation #11] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-14-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-773788190_22 at /127.0.0.1:52182 [Waiting for operation #5] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-14-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-773788190_22 at /127.0.0.1:40606 [Waiting for operation #14] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=927 (was 865) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=241 (was 228) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=7970 (was 8108) 2024-12-03T23:24:53,706 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayCompressed#testSequentialEditLogSeqNum Thread=407, OpenFileDescriptor=927, MaxFileDescriptor=1048576, SystemLoadAverage=241, ProcessCount=11, AvailableMemoryMB=7969 2024-12-03T23:24:53,722 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-03T23:24:53,728 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:35563/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733268293721, archiveDir=hdfs://localhost:35563/hbase/oldWALs, maxLogs=32 2024-12-03T23:24:53,729 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733268293729 2024-12-03T23:24:53,736 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733268293721/wal.1733268293729 2024-12-03T23:24:53,738 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new MockWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34451:34451),(127.0.0.1/127.0.0.1:36361:36361),(127.0.0.1/127.0.0.1:38065:38065)] 2024-12-03T23:24:53,739 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 7753bcf08319dd2d65920527a8400afd, NAME => 'testSequentialEditLogSeqNum,,1733268293722.7753bcf08319dd2d65920527a8400afd.', STARTKEY => '', ENDKEY => ''} 2024-12-03T23:24:53,739 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testSequentialEditLogSeqNum,,1733268293722.7753bcf08319dd2d65920527a8400afd.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T23:24:53,739 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 7753bcf08319dd2d65920527a8400afd 2024-12-03T23:24:53,739 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 7753bcf08319dd2d65920527a8400afd 2024-12-03T23:24:53,740 WARN [Time-limited test {}] regionserver.HRegionFileSystem(836): hdfs://localhost:35563/hbase/data/default/testSequentialEditLogSeqNum/7753bcf08319dd2d65920527a8400afd doesn't exist for region: 7753bcf08319dd2d65920527a8400afd on table testSequentialEditLogSeqNum 2024-12-03T23:24:53,741 WARN [Time-limited test {}] regionserver.HRegionFileSystem(854): .regioninfo file not found for region: 7753bcf08319dd2d65920527a8400afd on table testSequentialEditLogSeqNum 2024-12-03T23:24:53,748 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45491 is added to blk_1073741875_1052 (size=62) 2024-12-03T23:24:53,749 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41487 is added to blk_1073741875_1052 (size=62) 2024-12-03T23:24:53,749 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39829 is added to blk_1073741875_1052 (size=62) 2024-12-03T23:24:53,753 INFO [StoreOpener-7753bcf08319dd2d65920527a8400afd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 7753bcf08319dd2d65920527a8400afd 2024-12-03T23:24:53,755 INFO [StoreOpener-7753bcf08319dd2d65920527a8400afd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7753bcf08319dd2d65920527a8400afd columnFamilyName a 2024-12-03T23:24:53,755 DEBUG [StoreOpener-7753bcf08319dd2d65920527a8400afd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T23:24:53,755 INFO [StoreOpener-7753bcf08319dd2d65920527a8400afd-1 {}] regionserver.HStore(327): Store=7753bcf08319dd2d65920527a8400afd/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T23:24:53,756 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 7753bcf08319dd2d65920527a8400afd 2024-12-03T23:24:53,756 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35563/hbase/data/default/testSequentialEditLogSeqNum/7753bcf08319dd2d65920527a8400afd 2024-12-03T23:24:53,757 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35563/hbase/data/default/testSequentialEditLogSeqNum/7753bcf08319dd2d65920527a8400afd 2024-12-03T23:24:53,757 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 7753bcf08319dd2d65920527a8400afd 2024-12-03T23:24:53,757 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 7753bcf08319dd2d65920527a8400afd 2024-12-03T23:24:53,759 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 7753bcf08319dd2d65920527a8400afd 2024-12-03T23:24:53,761 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35563/hbase/data/default/testSequentialEditLogSeqNum/7753bcf08319dd2d65920527a8400afd/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T23:24:53,762 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 7753bcf08319dd2d65920527a8400afd; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71918460, jitterRate=0.07166856527328491}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-03T23:24:53,763 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 7753bcf08319dd2d65920527a8400afd: Writing region info on filesystem at 1733268293739Initializing all the Stores at 1733268293753 (+14 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733268293753Cleaning up temporary data from old regions at 1733268293757 (+4 ms)Region opened successfully at 1733268293763 (+6 ms) 2024-12-03T23:24:53,773 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 7753bcf08319dd2d65920527a8400afd 1/1 column families, dataSize=770 B heapSize=1.73 KB 2024-12-03T23:24:53,794 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35563/hbase/data/default/testSequentialEditLogSeqNum/7753bcf08319dd2d65920527a8400afd/.tmp/a/1566c9fe6d70449097144a4a876e1973 is 81, key is testSequentialEditLogSeqNum/a:x0/1733268293763/Put/seqid=0 2024-12-03T23:24:53,800 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39829 is added to blk_1073741876_1053 (size=5833) 2024-12-03T23:24:53,800 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45491 is added to blk_1073741876_1053 (size=5833) 2024-12-03T23:24:53,801 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41487 is added to blk_1073741876_1053 (size=5833) 2024-12-03T23:24:53,803 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=770 B at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:35563/hbase/data/default/testSequentialEditLogSeqNum/7753bcf08319dd2d65920527a8400afd/.tmp/a/1566c9fe6d70449097144a4a876e1973 2024-12-03T23:24:53,809 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35563/hbase/data/default/testSequentialEditLogSeqNum/7753bcf08319dd2d65920527a8400afd/.tmp/a/1566c9fe6d70449097144a4a876e1973 as hdfs://localhost:35563/hbase/data/default/testSequentialEditLogSeqNum/7753bcf08319dd2d65920527a8400afd/a/1566c9fe6d70449097144a4a876e1973 2024-12-03T23:24:53,815 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35563/hbase/data/default/testSequentialEditLogSeqNum/7753bcf08319dd2d65920527a8400afd/a/1566c9fe6d70449097144a4a876e1973, entries=10, sequenceid=13, filesize=5.7 K 2024-12-03T23:24:53,817 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~770 B/770, heapSize ~1.72 KB/1760, currentSize=0 B/0 for 7753bcf08319dd2d65920527a8400afd in 44ms, sequenceid=13, compaction requested=false 2024-12-03T23:24:53,817 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 7753bcf08319dd2d65920527a8400afd: 2024-12-03T23:24:53,822 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-03T23:24:53,822 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-03T23:24:53,822 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-03T23:24:53,822 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-03T23:24:53,823 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-03T23:24:53,824 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39829 is added to blk_1073741874_1051 (size=1616) 2024-12-03T23:24:53,825 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41487 is added to blk_1073741874_1051 (size=1616) 2024-12-03T23:24:53,825 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45491 is added to blk_1073741874_1051 (size=1616) 2024-12-03T23:24:53,841 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:35563/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733268293721/wal.1733268293729, size=1.6 K (1616bytes) 2024-12-03T23:24:53,841 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:35563/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733268293721/wal.1733268293729 2024-12-03T23:24:53,841 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:35563/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733268293721/wal.1733268293729 after 0ms 2024-12-03T23:24:53,843 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:35563/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733268293721/wal.1733268293729: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-03T23:24:53,844 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:35563/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733268293721/wal.1733268293729 took 4ms 2024-12-03T23:24:53,846 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:35563/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733268293721/wal.1733268293729 so closing down 2024-12-03T23:24:53,846 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-03T23:24:53,847 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1733268293729.temp 2024-12-03T23:24:53,848 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:35563/hbase/data/default/testSequentialEditLogSeqNum/7753bcf08319dd2d65920527a8400afd/recovered.edits/0000000000000000003-wal.1733268293729.temp 2024-12-03T23:24:53,849 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-03T23:24:53,857 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45491 is added to blk_1073741877_1054 (size=1175) 2024-12-03T23:24:53,857 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39829 is added to blk_1073741877_1054 (size=1175) 2024-12-03T23:24:53,858 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41487 is added to blk_1073741877_1054 (size=1175) 2024-12-03T23:24:53,860 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:35563/hbase/data/default/testSequentialEditLogSeqNum/7753bcf08319dd2d65920527a8400afd/recovered.edits/0000000000000000003-wal.1733268293729.temp (wrote 15 edits, skipped 0 edits in 0 ms) 2024-12-03T23:24:53,862 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:35563/hbase/data/default/testSequentialEditLogSeqNum/7753bcf08319dd2d65920527a8400afd/recovered.edits/0000000000000000003-wal.1733268293729.temp to hdfs://localhost:35563/hbase/data/default/testSequentialEditLogSeqNum/7753bcf08319dd2d65920527a8400afd/recovered.edits/0000000000000000020 2024-12-03T23:24:53,862 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 17 edits across 1 Regions in 18 ms; skipped=2; WAL=hdfs://localhost:35563/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733268293721/wal.1733268293729, size=1.6 K, length=1616, corrupted=false, cancelled=false 2024-12-03T23:24:53,862 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:35563/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733268293721/wal.1733268293729, journal: Splitting hdfs://localhost:35563/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733268293721/wal.1733268293729, size=1.6 K (1616bytes) at 1733268293841Finishing writing output for hdfs://localhost:35563/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733268293721/wal.1733268293729 so closing down at 1733268293846 (+5 ms)Creating recovered edits writer path=hdfs://localhost:35563/hbase/data/default/testSequentialEditLogSeqNum/7753bcf08319dd2d65920527a8400afd/recovered.edits/0000000000000000003-wal.1733268293729.temp at 1733268293848 (+2 ms)3 split writer threads finished at 1733268293849 (+1 ms)Closed recovered edits writer path=hdfs://localhost:35563/hbase/data/default/testSequentialEditLogSeqNum/7753bcf08319dd2d65920527a8400afd/recovered.edits/0000000000000000003-wal.1733268293729.temp (wrote 15 edits, skipped 0 edits in 0 ms) at 1733268293860 (+11 ms)Rename recovered edits hdfs://localhost:35563/hbase/data/default/testSequentialEditLogSeqNum/7753bcf08319dd2d65920527a8400afd/recovered.edits/0000000000000000003-wal.1733268293729.temp to hdfs://localhost:35563/hbase/data/default/testSequentialEditLogSeqNum/7753bcf08319dd2d65920527a8400afd/recovered.edits/0000000000000000020 at 1733268293862 (+2 ms)Processed 17 edits across 1 Regions in 18 ms; skipped=2; WAL=hdfs://localhost:35563/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733268293721/wal.1733268293729, size=1.6 K, length=1616, corrupted=false, cancelled=false at 1733268293862 2024-12-03T23:24:53,876 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayCompressed#testSequentialEditLogSeqNum Thread=412 (was 407) Potentially hanging thread: sync.4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-773788190_22 at /127.0.0.1:52182 [Waiting for operation #9] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: sync.0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-773788190_22 at /127.0.0.1:40606 [Waiting for operation #16] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: sync.1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) - Thread LEAK? -, OpenFileDescriptor=963 (was 927) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=241 (was 241), ProcessCount=11 (was 11), AvailableMemoryMB=7968 (was 7969) 2024-12-03T23:24:53,888 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayCompressed#testRegionMadeOfBulkLoadedFilesOnly Thread=412, OpenFileDescriptor=963, MaxFileDescriptor=1048576, SystemLoadAverage=241, ProcessCount=11, AvailableMemoryMB=7967 2024-12-03T23:24:53,903 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-03T23:24:53,905 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-03T23:24:53,925 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-03T23:24:53,927 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-39638089, suffix=, logDir=hdfs://localhost:35563/hbase/WALs/hregion-39638089, archiveDir=hdfs://localhost:35563/hbase/oldWALs, maxLogs=32 2024-12-03T23:24:53,947 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-39638089/hregion-39638089.1733268293928, exclude list is [], retry=0 2024-12-03T23:24:53,950 DEBUG [AsyncFSWAL-17-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39829,DS-e6270720-ef81-458e-b5d6-959d16c4c378,DISK] 2024-12-03T23:24:53,950 DEBUG [AsyncFSWAL-17-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45491,DS-d55e5535-c0ce-4dc2-a45c-8e70eb010e56,DISK] 2024-12-03T23:24:53,950 DEBUG [AsyncFSWAL-17-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41487,DS-459d168f-61f7-403b-9139-91eca369c54e,DISK] 2024-12-03T23:24:53,953 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-39638089/hregion-39638089.1733268293928 2024-12-03T23:24:53,953 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:38065:38065),(127.0.0.1/127.0.0.1:34451:34451),(127.0.0.1/127.0.0.1:36361:36361)] 2024-12-03T23:24:53,953 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => b210660de159ca8bd3187dac917abb56, NAME => 'testRegionMadeOfBulkLoadedFilesOnly,,1733268293903.b210660de159ca8bd3187dac917abb56.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testRegionMadeOfBulkLoadedFilesOnly', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35563/hbase 2024-12-03T23:24:53,963 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39829 is added to blk_1073741879_1056 (size=70) 2024-12-03T23:24:53,963 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45491 is added to blk_1073741879_1056 (size=70) 2024-12-03T23:24:53,964 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41487 is added to blk_1073741879_1056 (size=70) 2024-12-03T23:24:53,964 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testRegionMadeOfBulkLoadedFilesOnly,,1733268293903.b210660de159ca8bd3187dac917abb56.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T23:24:53,966 INFO [StoreOpener-b210660de159ca8bd3187dac917abb56-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region b210660de159ca8bd3187dac917abb56 2024-12-03T23:24:53,967 INFO [StoreOpener-b210660de159ca8bd3187dac917abb56-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b210660de159ca8bd3187dac917abb56 columnFamilyName a 2024-12-03T23:24:53,967 DEBUG [StoreOpener-b210660de159ca8bd3187dac917abb56-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T23:24:53,968 INFO [StoreOpener-b210660de159ca8bd3187dac917abb56-1 {}] regionserver.HStore(327): Store=b210660de159ca8bd3187dac917abb56/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T23:24:53,968 INFO [StoreOpener-b210660de159ca8bd3187dac917abb56-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region b210660de159ca8bd3187dac917abb56 2024-12-03T23:24:53,969 INFO [StoreOpener-b210660de159ca8bd3187dac917abb56-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b210660de159ca8bd3187dac917abb56 columnFamilyName b 2024-12-03T23:24:53,969 DEBUG [StoreOpener-b210660de159ca8bd3187dac917abb56-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T23:24:53,970 INFO [StoreOpener-b210660de159ca8bd3187dac917abb56-1 {}] regionserver.HStore(327): Store=b210660de159ca8bd3187dac917abb56/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T23:24:53,970 INFO [StoreOpener-b210660de159ca8bd3187dac917abb56-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region b210660de159ca8bd3187dac917abb56 2024-12-03T23:24:53,971 INFO [StoreOpener-b210660de159ca8bd3187dac917abb56-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b210660de159ca8bd3187dac917abb56 columnFamilyName c 2024-12-03T23:24:53,971 DEBUG [StoreOpener-b210660de159ca8bd3187dac917abb56-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T23:24:53,972 INFO [StoreOpener-b210660de159ca8bd3187dac917abb56-1 {}] regionserver.HStore(327): Store=b210660de159ca8bd3187dac917abb56/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T23:24:53,972 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for b210660de159ca8bd3187dac917abb56 2024-12-03T23:24:53,973 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35563/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/b210660de159ca8bd3187dac917abb56 2024-12-03T23:24:53,973 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35563/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/b210660de159ca8bd3187dac917abb56 2024-12-03T23:24:53,974 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for b210660de159ca8bd3187dac917abb56 2024-12-03T23:24:53,974 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for b210660de159ca8bd3187dac917abb56 2024-12-03T23:24:53,975 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRegionMadeOfBulkLoadedFilesOnly descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-03T23:24:53,977 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for b210660de159ca8bd3187dac917abb56 2024-12-03T23:24:53,979 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35563/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/b210660de159ca8bd3187dac917abb56/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T23:24:53,979 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened b210660de159ca8bd3187dac917abb56; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63315723, jitterRate=-0.05652220547199249}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-03T23:24:53,981 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for b210660de159ca8bd3187dac917abb56: Writing region info on filesystem at 1733268293964Initializing all the Stores at 1733268293965 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733268293965Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733268293965Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733268293965Cleaning up temporary data from old regions at 1733268293974 (+9 ms)Region opened successfully at 1733268293981 (+7 ms) 2024-12-03T23:24:53,981 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing b210660de159ca8bd3187dac917abb56, disabling compactions & flushes 2024-12-03T23:24:53,981 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testRegionMadeOfBulkLoadedFilesOnly,,1733268293903.b210660de159ca8bd3187dac917abb56. 2024-12-03T23:24:53,981 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testRegionMadeOfBulkLoadedFilesOnly,,1733268293903.b210660de159ca8bd3187dac917abb56. 2024-12-03T23:24:53,981 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testRegionMadeOfBulkLoadedFilesOnly,,1733268293903.b210660de159ca8bd3187dac917abb56. after waiting 0 ms 2024-12-03T23:24:53,981 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testRegionMadeOfBulkLoadedFilesOnly,,1733268293903.b210660de159ca8bd3187dac917abb56. 2024-12-03T23:24:53,982 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testRegionMadeOfBulkLoadedFilesOnly,,1733268293903.b210660de159ca8bd3187dac917abb56. 2024-12-03T23:24:53,982 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for b210660de159ca8bd3187dac917abb56: Waiting for close lock at 1733268293981Disabling compacts and flushes for region at 1733268293981Disabling writes for close at 1733268293981Writing region close event to WAL at 1733268293982 (+1 ms)Closed at 1733268293982 2024-12-03T23:24:53,985 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41487 is added to blk_1073741878_1055 (size=93) 2024-12-03T23:24:53,985 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39829 is added to blk_1073741878_1055 (size=93) 2024-12-03T23:24:53,985 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45491 is added to blk_1073741878_1055 (size=93) 2024-12-03T23:24:53,987 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-03T23:24:53,987 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-39638089:(num 1733268293928) 2024-12-03T23:24:53,987 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-03T23:24:53,989 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:35563/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733268293902, archiveDir=hdfs://localhost:35563/hbase/oldWALs, maxLogs=32 2024-12-03T23:24:54,008 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733268293902/wal.1733268293990, exclude list is [], retry=0 2024-12-03T23:24:54,012 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39829,DS-e6270720-ef81-458e-b5d6-959d16c4c378,DISK] 2024-12-03T23:24:54,012 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41487,DS-459d168f-61f7-403b-9139-91eca369c54e,DISK] 2024-12-03T23:24:54,012 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45491,DS-d55e5535-c0ce-4dc2-a45c-8e70eb010e56,DISK] 2024-12-03T23:24:54,014 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733268293902/wal.1733268293990 2024-12-03T23:24:54,014 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:38065:38065),(127.0.0.1/127.0.0.1:36361:36361),(127.0.0.1/127.0.0.1:34451:34451)] 2024-12-03T23:24:54,015 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => b210660de159ca8bd3187dac917abb56, NAME => 'testRegionMadeOfBulkLoadedFilesOnly,,1733268293903.b210660de159ca8bd3187dac917abb56.', STARTKEY => '', ENDKEY => ''} 2024-12-03T23:24:54,015 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testRegionMadeOfBulkLoadedFilesOnly,,1733268293903.b210660de159ca8bd3187dac917abb56.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T23:24:54,015 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for b210660de159ca8bd3187dac917abb56 2024-12-03T23:24:54,015 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for b210660de159ca8bd3187dac917abb56 2024-12-03T23:24:54,016 INFO [StoreOpener-b210660de159ca8bd3187dac917abb56-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region b210660de159ca8bd3187dac917abb56 2024-12-03T23:24:54,017 INFO [StoreOpener-b210660de159ca8bd3187dac917abb56-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b210660de159ca8bd3187dac917abb56 columnFamilyName a 2024-12-03T23:24:54,017 DEBUG [StoreOpener-b210660de159ca8bd3187dac917abb56-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T23:24:54,018 INFO [StoreOpener-b210660de159ca8bd3187dac917abb56-1 {}] regionserver.HStore(327): Store=b210660de159ca8bd3187dac917abb56/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T23:24:54,018 INFO [StoreOpener-b210660de159ca8bd3187dac917abb56-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region b210660de159ca8bd3187dac917abb56 2024-12-03T23:24:54,019 INFO [StoreOpener-b210660de159ca8bd3187dac917abb56-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b210660de159ca8bd3187dac917abb56 columnFamilyName b 2024-12-03T23:24:54,019 DEBUG [StoreOpener-b210660de159ca8bd3187dac917abb56-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T23:24:54,019 INFO [StoreOpener-b210660de159ca8bd3187dac917abb56-1 {}] regionserver.HStore(327): Store=b210660de159ca8bd3187dac917abb56/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T23:24:54,019 INFO [StoreOpener-b210660de159ca8bd3187dac917abb56-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region b210660de159ca8bd3187dac917abb56 2024-12-03T23:24:54,020 INFO [StoreOpener-b210660de159ca8bd3187dac917abb56-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b210660de159ca8bd3187dac917abb56 columnFamilyName c 2024-12-03T23:24:54,020 DEBUG [StoreOpener-b210660de159ca8bd3187dac917abb56-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T23:24:54,021 INFO [StoreOpener-b210660de159ca8bd3187dac917abb56-1 {}] regionserver.HStore(327): Store=b210660de159ca8bd3187dac917abb56/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T23:24:54,021 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for b210660de159ca8bd3187dac917abb56 2024-12-03T23:24:54,021 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35563/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/b210660de159ca8bd3187dac917abb56 2024-12-03T23:24:54,022 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35563/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/b210660de159ca8bd3187dac917abb56 2024-12-03T23:24:54,023 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for b210660de159ca8bd3187dac917abb56 2024-12-03T23:24:54,023 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for b210660de159ca8bd3187dac917abb56 2024-12-03T23:24:54,024 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRegionMadeOfBulkLoadedFilesOnly descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-03T23:24:54,025 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for b210660de159ca8bd3187dac917abb56 2024-12-03T23:24:54,026 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened b210660de159ca8bd3187dac917abb56; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62472712, jitterRate=-0.0690840482711792}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-03T23:24:54,027 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for b210660de159ca8bd3187dac917abb56: Writing region info on filesystem at 1733268294015Initializing all the Stores at 1733268294016 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733268294016Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733268294016Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733268294016Cleaning up temporary data from old regions at 1733268294023 (+7 ms)Region opened successfully at 1733268294027 (+4 ms) 2024-12-03T23:24:54,031 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35563/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile is 28, key is \x0D/a:a/1733268294030/Put/seqid=0 2024-12-03T23:24:54,039 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45491 is added to blk_1073741881_1058 (size=4826) 2024-12-03T23:24:54,040 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39829 is added to blk_1073741881_1058 (size=4826) 2024-12-03T23:24:54,040 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41487 is added to blk_1073741881_1058 (size=4826) 2024-12-03T23:24:54,042 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:35563/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile for inclusion in b210660de159ca8bd3187dac917abb56/a 2024-12-03T23:24:54,050 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first= last=z 2024-12-03T23:24:54,050 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-12-03T23:24:54,051 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for b210660de159ca8bd3187dac917abb56: 2024-12-03T23:24:54,052 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35563/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile as hdfs://localhost:35563/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/b210660de159ca8bd3187dac917abb56/a/653f002fd51b486388ceab64e0320880_SeqId_3_ 2024-12-03T23:24:54,053 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:35563/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile into b210660de159ca8bd3187dac917abb56/a as hdfs://localhost:35563/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/b210660de159ca8bd3187dac917abb56/a/653f002fd51b486388ceab64e0320880_SeqId_3_ - updating store file list. 2024-12-03T23:24:54,058 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for 653f002fd51b486388ceab64e0320880_SeqId_3_: NONE, but ROW specified in column family configuration 2024-12-03T23:24:54,058 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:35563/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/b210660de159ca8bd3187dac917abb56/a/653f002fd51b486388ceab64e0320880_SeqId_3_ into b210660de159ca8bd3187dac917abb56/a 2024-12-03T23:24:54,059 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:35563/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile into b210660de159ca8bd3187dac917abb56/a (new location: hdfs://localhost:35563/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/b210660de159ca8bd3187dac917abb56/a/653f002fd51b486388ceab64e0320880_SeqId_3_) 2024-12-03T23:24:54,098 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:35563/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733268293902/wal.1733268293990, size=0 (0bytes) 2024-12-03T23:24:54,098 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:35563/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733268293902/wal.1733268293990 might be still open, length is 0 2024-12-03T23:24:54,098 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:35563/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733268293902/wal.1733268293990 2024-12-03T23:24:54,099 WARN [IPC Server handler 1 on default port 35563 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733268293902/wal.1733268293990 has not been closed. Lease recovery is in progress. RecoveryId = 1059 for block blk_1073741880_1057 2024-12-03T23:24:54,099 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:35563/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733268293902/wal.1733268293990 after 1ms 2024-12-03T23:24:56,771 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-773788190_22 at /127.0.0.1:52278 [Receiving block BP-2057003906-172.17.0.2-1733268279817:blk_1073741880_1057] {}] datanode.DataXceiver(331): 127.0.0.1:39829:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:52278 dst: /127.0.0.1:39829 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:39829 remote=/127.0.0.1:52278]. Total timeout mills is 60000, 57293 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T23:24:56,771 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-773788190_22 at /127.0.0.1:43992 [Receiving block BP-2057003906-172.17.0.2-1733268279817:blk_1073741880_1057] {}] datanode.DataXceiver(331): 127.0.0.1:41487:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:43992 dst: /127.0.0.1:41487 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T23:24:56,771 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-773788190_22 at /127.0.0.1:40800 [Receiving block BP-2057003906-172.17.0.2-1733268279817:blk_1073741880_1057] {}] datanode.DataXceiver(331): 127.0.0.1:45491:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:40800 dst: /127.0.0.1:45491 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T23:24:56,774 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39829 is added to blk_1073741880_1059 (size=434) 2024-12-03T23:24:56,774 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41487 is added to blk_1073741880_1059 (size=434) 2024-12-03T23:24:56,775 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45491 is added to blk_1073741880_1059 (size=434) 2024-12-03T23:24:58,100 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:35563/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733268293902/wal.1733268293990 after 4002ms 2024-12-03T23:24:58,103 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:35563/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733268293902/wal.1733268293990: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-03T23:24:58,104 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:35563/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733268293902/wal.1733268293990 took 4006ms 2024-12-03T23:24:58,105 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:35563/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733268293902/wal.1733268293990; continuing. 2024-12-03T23:24:58,106 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:35563/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733268293902/wal.1733268293990 so closing down 2024-12-03T23:24:58,106 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-03T23:24:58,107 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000005-wal.1733268293990.temp 2024-12-03T23:24:58,109 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:35563/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/b210660de159ca8bd3187dac917abb56/recovered.edits/0000000000000000005-wal.1733268293990.temp 2024-12-03T23:24:58,109 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-03T23:24:58,115 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39829 is added to blk_1073741882_1060 (size=236) 2024-12-03T23:24:58,115 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41487 is added to blk_1073741882_1060 (size=236) 2024-12-03T23:24:58,115 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45491 is added to blk_1073741882_1060 (size=236) 2024-12-03T23:24:58,115 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:35563/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/b210660de159ca8bd3187dac917abb56/recovered.edits/0000000000000000005-wal.1733268293990.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-12-03T23:24:58,117 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:35563/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/b210660de159ca8bd3187dac917abb56/recovered.edits/0000000000000000005-wal.1733268293990.temp to hdfs://localhost:35563/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/b210660de159ca8bd3187dac917abb56/recovered.edits/0000000000000000005 2024-12-03T23:24:58,117 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 2 edits across 1 Regions in 13 ms; skipped=1; WAL=hdfs://localhost:35563/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733268293902/wal.1733268293990, size=0, length=0, corrupted=false, cancelled=false 2024-12-03T23:24:58,117 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:35563/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733268293902/wal.1733268293990, journal: Splitting hdfs://localhost:35563/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733268293902/wal.1733268293990, size=0 (0bytes) at 1733268294098Finishing writing output for hdfs://localhost:35563/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733268293902/wal.1733268293990 so closing down at 1733268298106 (+4008 ms)Creating recovered edits writer path=hdfs://localhost:35563/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/b210660de159ca8bd3187dac917abb56/recovered.edits/0000000000000000005-wal.1733268293990.temp at 1733268298109 (+3 ms)3 split writer threads finished at 1733268298109Closed recovered edits writer path=hdfs://localhost:35563/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/b210660de159ca8bd3187dac917abb56/recovered.edits/0000000000000000005-wal.1733268293990.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1733268298115 (+6 ms)Rename recovered edits hdfs://localhost:35563/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/b210660de159ca8bd3187dac917abb56/recovered.edits/0000000000000000005-wal.1733268293990.temp to hdfs://localhost:35563/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/b210660de159ca8bd3187dac917abb56/recovered.edits/0000000000000000005 at 1733268298117 (+2 ms)Processed 2 edits across 1 Regions in 13 ms; skipped=1; WAL=hdfs://localhost:35563/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733268293902/wal.1733268293990, size=0, length=0, corrupted=false, cancelled=false at 1733268298117 2024-12-03T23:24:58,119 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:35563/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733268293902/wal.1733268293990 to hdfs://localhost:35563/hbase/oldWALs/wal.1733268293990 2024-12-03T23:24:58,120 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:35563/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/b210660de159ca8bd3187dac917abb56/recovered.edits/0000000000000000005 2024-12-03T23:24:58,120 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-03T23:24:58,122 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:35563/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733268293902, archiveDir=hdfs://localhost:35563/hbase/oldWALs, maxLogs=32 2024-12-03T23:24:58,135 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733268293902/wal.1733268298122, exclude list is [], retry=0 2024-12-03T23:24:58,138 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39829,DS-e6270720-ef81-458e-b5d6-959d16c4c378,DISK] 2024-12-03T23:24:58,139 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41487,DS-459d168f-61f7-403b-9139-91eca369c54e,DISK] 2024-12-03T23:24:58,139 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45491,DS-d55e5535-c0ce-4dc2-a45c-8e70eb010e56,DISK] 2024-12-03T23:24:58,141 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733268293902/wal.1733268298122 2024-12-03T23:24:58,141 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:38065:38065),(127.0.0.1/127.0.0.1:36361:36361),(127.0.0.1/127.0.0.1:34451:34451)] 2024-12-03T23:24:58,141 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => b210660de159ca8bd3187dac917abb56, NAME => 'testRegionMadeOfBulkLoadedFilesOnly,,1733268293903.b210660de159ca8bd3187dac917abb56.', STARTKEY => '', ENDKEY => ''} 2024-12-03T23:24:58,142 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testRegionMadeOfBulkLoadedFilesOnly,,1733268293903.b210660de159ca8bd3187dac917abb56.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T23:24:58,142 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for b210660de159ca8bd3187dac917abb56 2024-12-03T23:24:58,142 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for b210660de159ca8bd3187dac917abb56 2024-12-03T23:24:58,143 INFO [StoreOpener-b210660de159ca8bd3187dac917abb56-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region b210660de159ca8bd3187dac917abb56 2024-12-03T23:24:58,144 INFO [StoreOpener-b210660de159ca8bd3187dac917abb56-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b210660de159ca8bd3187dac917abb56 columnFamilyName a 2024-12-03T23:24:58,144 DEBUG [StoreOpener-b210660de159ca8bd3187dac917abb56-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T23:24:58,151 DEBUG [StoreFileOpener-b210660de159ca8bd3187dac917abb56-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for 653f002fd51b486388ceab64e0320880_SeqId_3_: NONE, but ROW specified in column family configuration 2024-12-03T23:24:58,151 DEBUG [StoreOpener-b210660de159ca8bd3187dac917abb56-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:35563/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/b210660de159ca8bd3187dac917abb56/a/653f002fd51b486388ceab64e0320880_SeqId_3_ 2024-12-03T23:24:58,151 INFO [StoreOpener-b210660de159ca8bd3187dac917abb56-1 {}] regionserver.HStore(327): Store=b210660de159ca8bd3187dac917abb56/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T23:24:58,151 INFO [StoreOpener-b210660de159ca8bd3187dac917abb56-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region b210660de159ca8bd3187dac917abb56 2024-12-03T23:24:58,152 INFO [StoreOpener-b210660de159ca8bd3187dac917abb56-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b210660de159ca8bd3187dac917abb56 columnFamilyName b 2024-12-03T23:24:58,152 DEBUG [StoreOpener-b210660de159ca8bd3187dac917abb56-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T23:24:58,153 INFO [StoreOpener-b210660de159ca8bd3187dac917abb56-1 {}] regionserver.HStore(327): Store=b210660de159ca8bd3187dac917abb56/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T23:24:58,153 INFO [StoreOpener-b210660de159ca8bd3187dac917abb56-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region b210660de159ca8bd3187dac917abb56 2024-12-03T23:24:58,154 INFO [StoreOpener-b210660de159ca8bd3187dac917abb56-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b210660de159ca8bd3187dac917abb56 columnFamilyName c 2024-12-03T23:24:58,154 DEBUG [StoreOpener-b210660de159ca8bd3187dac917abb56-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T23:24:58,154 INFO [StoreOpener-b210660de159ca8bd3187dac917abb56-1 {}] regionserver.HStore(327): Store=b210660de159ca8bd3187dac917abb56/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T23:24:58,154 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for b210660de159ca8bd3187dac917abb56 2024-12-03T23:24:58,155 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35563/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/b210660de159ca8bd3187dac917abb56 2024-12-03T23:24:58,156 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:35563/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/b210660de159ca8bd3187dac917abb56 2024-12-03T23:24:58,157 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:35563/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/b210660de159ca8bd3187dac917abb56/recovered.edits/0000000000000000005 2024-12-03T23:24:58,159 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:35563/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/b210660de159ca8bd3187dac917abb56/recovered.edits/0000000000000000005: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-03T23:24:58,160 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 1, skipped 0, firstSequenceIdInLog=5, maxSequenceIdInLog=5, path=hdfs://localhost:35563/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/b210660de159ca8bd3187dac917abb56/recovered.edits/0000000000000000005 2024-12-03T23:24:58,160 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing b210660de159ca8bd3187dac917abb56 3/3 column families, dataSize=58 B heapSize=904 B 2024-12-03T23:24:58,175 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35563/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/b210660de159ca8bd3187dac917abb56/.tmp/a/5814512d74b64d81ac19122ddb8fd2c8 is 62, key is testRegionMadeOfBulkLoadedFilesOnly/a:a/1733268294063/Put/seqid=0 2024-12-03T23:24:58,181 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45491 is added to blk_1073741884_1062 (size=5149) 2024-12-03T23:24:58,181 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39829 is added to blk_1073741884_1062 (size=5149) 2024-12-03T23:24:58,182 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41487 is added to blk_1073741884_1062 (size=5149) 2024-12-03T23:24:58,182 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58 B at sequenceid=5 (bloomFilter=true), to=hdfs://localhost:35563/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/b210660de159ca8bd3187dac917abb56/.tmp/a/5814512d74b64d81ac19122ddb8fd2c8 2024-12-03T23:24:58,189 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35563/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/b210660de159ca8bd3187dac917abb56/.tmp/a/5814512d74b64d81ac19122ddb8fd2c8 as hdfs://localhost:35563/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/b210660de159ca8bd3187dac917abb56/a/5814512d74b64d81ac19122ddb8fd2c8 2024-12-03T23:24:58,194 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35563/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/b210660de159ca8bd3187dac917abb56/a/5814512d74b64d81ac19122ddb8fd2c8, entries=1, sequenceid=5, filesize=5.0 K 2024-12-03T23:24:58,194 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~58 B/58, heapSize ~376 B/376, currentSize=0 B/0 for b210660de159ca8bd3187dac917abb56 in 34ms, sequenceid=5, compaction requested=false; wal=null 2024-12-03T23:24:58,195 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:35563/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/b210660de159ca8bd3187dac917abb56/recovered.edits/0000000000000000005 2024-12-03T23:24:58,196 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for b210660de159ca8bd3187dac917abb56 2024-12-03T23:24:58,196 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for b210660de159ca8bd3187dac917abb56 2024-12-03T23:24:58,197 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRegionMadeOfBulkLoadedFilesOnly descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-03T23:24:58,198 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for b210660de159ca8bd3187dac917abb56 2024-12-03T23:24:58,200 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35563/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/b210660de159ca8bd3187dac917abb56/recovered.edits/5.seqid, newMaxSeqId=5, maxSeqId=1 2024-12-03T23:24:58,201 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened b210660de159ca8bd3187dac917abb56; next sequenceid=6; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72747789, jitterRate=0.0840265303850174}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-03T23:24:58,201 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for b210660de159ca8bd3187dac917abb56: Writing region info on filesystem at 1733268298142Initializing all the Stores at 1733268298143 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733268298143Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733268298143Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733268298143Obtaining lock to block concurrent updates at 1733268298160 (+17 ms)Preparing flush snapshotting stores in b210660de159ca8bd3187dac917abb56 at 1733268298160Finished memstore snapshotting testRegionMadeOfBulkLoadedFilesOnly,,1733268293903.b210660de159ca8bd3187dac917abb56., syncing WAL and waiting on mvcc, flushsize=dataSize=58, getHeapSize=856, getOffHeapSize=0, getCellsCount=1 at 1733268298160Flushing stores of testRegionMadeOfBulkLoadedFilesOnly,,1733268293903.b210660de159ca8bd3187dac917abb56. at 1733268298160Flushing b210660de159ca8bd3187dac917abb56/a: creating writer at 1733268298160Flushing b210660de159ca8bd3187dac917abb56/a: appending metadata at 1733268298174 (+14 ms)Flushing b210660de159ca8bd3187dac917abb56/a: closing flushed file at 1733268298174Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7e749b6: reopening flushed file at 1733268298187 (+13 ms)Finished flush of dataSize ~58 B/58, heapSize ~376 B/376, currentSize=0 B/0 for b210660de159ca8bd3187dac917abb56 in 34ms, sequenceid=5, compaction requested=false; wal=null at 1733268298194 (+7 ms)Cleaning up temporary data from old regions at 1733268298196 (+2 ms)Region opened successfully at 1733268298201 (+5 ms) 2024-12-03T23:24:58,205 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing b210660de159ca8bd3187dac917abb56, disabling compactions & flushes 2024-12-03T23:24:58,205 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testRegionMadeOfBulkLoadedFilesOnly,,1733268293903.b210660de159ca8bd3187dac917abb56. 2024-12-03T23:24:58,205 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testRegionMadeOfBulkLoadedFilesOnly,,1733268293903.b210660de159ca8bd3187dac917abb56. 2024-12-03T23:24:58,205 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testRegionMadeOfBulkLoadedFilesOnly,,1733268293903.b210660de159ca8bd3187dac917abb56. after waiting 0 ms 2024-12-03T23:24:58,205 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testRegionMadeOfBulkLoadedFilesOnly,,1733268293903.b210660de159ca8bd3187dac917abb56. 2024-12-03T23:24:58,206 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testRegionMadeOfBulkLoadedFilesOnly,,1733268293903.b210660de159ca8bd3187dac917abb56. 2024-12-03T23:24:58,206 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for b210660de159ca8bd3187dac917abb56: Waiting for close lock at 1733268298205Disabling compacts and flushes for region at 1733268298205Disabling writes for close at 1733268298205Writing region close event to WAL at 1733268298206 (+1 ms)Closed at 1733268298206 2024-12-03T23:24:58,209 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41487 is added to blk_1073741883_1061 (size=93) 2024-12-03T23:24:58,209 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39829 is added to blk_1073741883_1061 (size=93) 2024-12-03T23:24:58,210 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45491 is added to blk_1073741883_1061 (size=93) 2024-12-03T23:24:58,212 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-03T23:24:58,212 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1733268298122) 2024-12-03T23:24:58,226 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayCompressed#testRegionMadeOfBulkLoadedFilesOnly Thread=413 (was 412) Potentially hanging thread: AsyncFSWAL-17-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-17-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1987708253) connection to localhost/127.0.0.1:35563 from jenkinstestRegionMadeOfBulkLoadedFilesOnly java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1301079526_22 at /127.0.0.1:44022 [Waiting for operation #3] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-17-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1301079526_22 at /127.0.0.1:52316 [Waiting for operation #13] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkinstestRegionMadeOfBulkLoadedFilesOnly@localhost:35563 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1022 (was 963) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=222 (was 241), ProcessCount=11 (was 11), AvailableMemoryMB=7969 (was 7967) - AvailableMemoryMB LEAK? - 2024-12-03T23:24:58,240 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayCompressed#testReplayEditsAfterRegionMovedWithMultiCF Thread=413, OpenFileDescriptor=1022, MaxFileDescriptor=1048576, SystemLoadAverage=222, ProcessCount=11, AvailableMemoryMB=7967 2024-12-03T23:24:58,256 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-03T23:24:58,261 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-03T23:24:58,265 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 5f9d64419733,41995,1733268282840 2024-12-03T23:24:58,268 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@415e15a8 2024-12-03T23:24:58,269 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-03T23:24:58,271 INFO [HMaster-EventLoopGroup-2-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50542, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-03T23:24:58,274 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41995 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testReplayEditsAfterRegionMovedWithMultiCF', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-03T23:24:58,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41995 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF 2024-12-03T23:24:58,283 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_PRE_OPERATION 2024-12-03T23:24:58,285 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41995 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testReplayEditsAfterRegionMovedWithMultiCF" procId is: 4 2024-12-03T23:24:58,285 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T23:24:58,286 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-03T23:24:58,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-03T23:24:58,296 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39829 is added to blk_1073741885_1063 (size=694) 2024-12-03T23:24:58,297 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41487 is added to blk_1073741885_1063 (size=694) 2024-12-03T23:24:58,297 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45491 is added to blk_1073741885_1063 (size=694) 2024-12-03T23:24:58,299 INFO [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 1745e7d6d10a6c6039d4c33aadf95a2f, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1733268298272.1745e7d6d10a6c6039d4c33aadf95a2f.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsAfterRegionMovedWithMultiCF', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35563/user/jenkins/test-data/e9061e95-7d9e-c12a-b652-9e8a1d87f0cf 2024-12-03T23:24:58,306 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41487 is added to blk_1073741886_1064 (size=77) 2024-12-03T23:24:58,306 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45491 is added to blk_1073741886_1064 (size=77) 2024-12-03T23:24:58,307 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39829 is added to blk_1073741886_1064 (size=77) 2024-12-03T23:24:58,308 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1733268298272.1745e7d6d10a6c6039d4c33aadf95a2f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T23:24:58,308 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1722): Closing 1745e7d6d10a6c6039d4c33aadf95a2f, disabling compactions & flushes 2024-12-03T23:24:58,308 INFO [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1733268298272.1745e7d6d10a6c6039d4c33aadf95a2f. 2024-12-03T23:24:58,308 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733268298272.1745e7d6d10a6c6039d4c33aadf95a2f. 2024-12-03T23:24:58,308 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733268298272.1745e7d6d10a6c6039d4c33aadf95a2f. after waiting 0 ms 2024-12-03T23:24:58,308 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1733268298272.1745e7d6d10a6c6039d4c33aadf95a2f. 2024-12-03T23:24:58,308 INFO [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733268298272.1745e7d6d10a6c6039d4c33aadf95a2f. 2024-12-03T23:24:58,308 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1676): Region close journal for 1745e7d6d10a6c6039d4c33aadf95a2f: Waiting for close lock at 1733268298308Disabling compacts and flushes for region at 1733268298308Disabling writes for close at 1733268298308Writing region close event to WAL at 1733268298308Closed at 1733268298308 2024-12-03T23:24:58,311 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_ADD_TO_META 2024-12-03T23:24:58,316 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testReplayEditsAfterRegionMovedWithMultiCF,,1733268298272.1745e7d6d10a6c6039d4c33aadf95a2f.","families":{"info":[{"qualifier":"regioninfo","vlen":76,"tag":[],"timestamp":"1733268298311"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733268298311"}]},"ts":"1733268298311"} 2024-12-03T23:24:58,320 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-03T23:24:58,322 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-03T23:24:58,324 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testReplayEditsAfterRegionMovedWithMultiCF","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733268298322"}]},"ts":"1733268298322"} 2024-12-03T23:24:58,329 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testReplayEditsAfterRegionMovedWithMultiCF, state=ENABLING in hbase:meta 2024-12-03T23:24:58,329 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {5f9d64419733=0} racks are {/default-rack=0} 2024-12-03T23:24:58,330 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-12-03T23:24:58,330 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-12-03T23:24:58,330 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-12-03T23:24:58,330 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-03T23:24:58,330 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-03T23:24:58,330 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-12-03T23:24:58,330 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-03T23:24:58,330 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-03T23:24:58,330 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-12-03T23:24:58,330 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-12-03T23:24:58,332 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=1745e7d6d10a6c6039d4c33aadf95a2f, ASSIGN}] 2024-12-03T23:24:58,334 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=1745e7d6d10a6c6039d4c33aadf95a2f, ASSIGN 2024-12-03T23:24:58,335 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=1745e7d6d10a6c6039d4c33aadf95a2f, ASSIGN; state=OFFLINE, location=5f9d64419733,40969,1733268283535; forceNewPlan=false, retain=false 2024-12-03T23:24:58,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-03T23:24:58,488 INFO [5f9d64419733:41995 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-12-03T23:24:58,489 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=1745e7d6d10a6c6039d4c33aadf95a2f, regionState=OPENING, regionLocation=5f9d64419733,40969,1733268283535 2024-12-03T23:24:58,493 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=1745e7d6d10a6c6039d4c33aadf95a2f, ASSIGN because future has completed 2024-12-03T23:24:58,494 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1745e7d6d10a6c6039d4c33aadf95a2f, server=5f9d64419733,40969,1733268283535}] 2024-12-03T23:24:58,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-03T23:24:58,652 INFO [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1733268298272.1745e7d6d10a6c6039d4c33aadf95a2f. 2024-12-03T23:24:58,653 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 1745e7d6d10a6c6039d4c33aadf95a2f, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1733268298272.1745e7d6d10a6c6039d4c33aadf95a2f.', STARTKEY => '', ENDKEY => ''} 2024-12-03T23:24:58,653 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF 1745e7d6d10a6c6039d4c33aadf95a2f 2024-12-03T23:24:58,653 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1733268298272.1745e7d6d10a6c6039d4c33aadf95a2f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T23:24:58,653 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 1745e7d6d10a6c6039d4c33aadf95a2f 2024-12-03T23:24:58,653 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 1745e7d6d10a6c6039d4c33aadf95a2f 2024-12-03T23:24:58,655 INFO [StoreOpener-1745e7d6d10a6c6039d4c33aadf95a2f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 1745e7d6d10a6c6039d4c33aadf95a2f 2024-12-03T23:24:58,656 INFO [StoreOpener-1745e7d6d10a6c6039d4c33aadf95a2f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1745e7d6d10a6c6039d4c33aadf95a2f columnFamilyName cf1 2024-12-03T23:24:58,656 DEBUG [StoreOpener-1745e7d6d10a6c6039d4c33aadf95a2f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T23:24:58,657 INFO [StoreOpener-1745e7d6d10a6c6039d4c33aadf95a2f-1 {}] regionserver.HStore(327): Store=1745e7d6d10a6c6039d4c33aadf95a2f/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T23:24:58,657 INFO [StoreOpener-1745e7d6d10a6c6039d4c33aadf95a2f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 1745e7d6d10a6c6039d4c33aadf95a2f 2024-12-03T23:24:58,658 INFO [StoreOpener-1745e7d6d10a6c6039d4c33aadf95a2f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1745e7d6d10a6c6039d4c33aadf95a2f columnFamilyName cf2 2024-12-03T23:24:58,658 DEBUG [StoreOpener-1745e7d6d10a6c6039d4c33aadf95a2f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T23:24:58,659 INFO [StoreOpener-1745e7d6d10a6c6039d4c33aadf95a2f-1 {}] regionserver.HStore(327): Store=1745e7d6d10a6c6039d4c33aadf95a2f/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T23:24:58,659 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 1745e7d6d10a6c6039d4c33aadf95a2f 2024-12-03T23:24:58,660 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35563/user/jenkins/test-data/e9061e95-7d9e-c12a-b652-9e8a1d87f0cf/data/default/testReplayEditsAfterRegionMovedWithMultiCF/1745e7d6d10a6c6039d4c33aadf95a2f 2024-12-03T23:24:58,660 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35563/user/jenkins/test-data/e9061e95-7d9e-c12a-b652-9e8a1d87f0cf/data/default/testReplayEditsAfterRegionMovedWithMultiCF/1745e7d6d10a6c6039d4c33aadf95a2f 2024-12-03T23:24:58,661 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 1745e7d6d10a6c6039d4c33aadf95a2f 2024-12-03T23:24:58,661 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 1745e7d6d10a6c6039d4c33aadf95a2f 2024-12-03T23:24:58,662 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-12-03T23:24:58,664 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 1745e7d6d10a6c6039d4c33aadf95a2f 2024-12-03T23:24:58,667 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35563/user/jenkins/test-data/e9061e95-7d9e-c12a-b652-9e8a1d87f0cf/data/default/testReplayEditsAfterRegionMovedWithMultiCF/1745e7d6d10a6c6039d4c33aadf95a2f/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T23:24:58,667 INFO [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 1745e7d6d10a6c6039d4c33aadf95a2f; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70608922, jitterRate=0.05215492844581604}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-12-03T23:24:58,667 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1745e7d6d10a6c6039d4c33aadf95a2f 2024-12-03T23:24:58,668 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 1745e7d6d10a6c6039d4c33aadf95a2f: Running coprocessor pre-open hook at 1733268298654Writing region info on filesystem at 1733268298654Initializing all the Stores at 1733268298654Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733268298654Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733268298655 (+1 ms)Cleaning up temporary data from old regions at 1733268298661 (+6 ms)Running coprocessor post-open hooks at 1733268298667 (+6 ms)Region opened successfully at 1733268298668 (+1 ms) 2024-12-03T23:24:58,670 INFO [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1733268298272.1745e7d6d10a6c6039d4c33aadf95a2f., pid=6, masterSystemTime=1733268298647 2024-12-03T23:24:58,674 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1733268298272.1745e7d6d10a6c6039d4c33aadf95a2f. 2024-12-03T23:24:58,674 INFO [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1733268298272.1745e7d6d10a6c6039d4c33aadf95a2f. 2024-12-03T23:24:58,675 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=1745e7d6d10a6c6039d4c33aadf95a2f, regionState=OPEN, openSeqNum=2, regionLocation=5f9d64419733,40969,1733268283535 2024-12-03T23:24:58,678 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1745e7d6d10a6c6039d4c33aadf95a2f, server=5f9d64419733,40969,1733268283535 because future has completed 2024-12-03T23:24:58,682 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-12-03T23:24:58,682 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 1745e7d6d10a6c6039d4c33aadf95a2f, server=5f9d64419733,40969,1733268283535 in 186 msec 2024-12-03T23:24:58,685 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-12-03T23:24:58,685 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=1745e7d6d10a6c6039d4c33aadf95a2f, ASSIGN in 350 msec 2024-12-03T23:24:58,687 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-03T23:24:58,687 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testReplayEditsAfterRegionMovedWithMultiCF","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733268298687"}]},"ts":"1733268298687"} 2024-12-03T23:24:58,690 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testReplayEditsAfterRegionMovedWithMultiCF, state=ENABLED in hbase:meta 2024-12-03T23:24:58,691 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_POST_OPERATION 2024-12-03T23:24:58,694 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF in 415 msec 2024-12-03T23:24:58,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-03T23:24:58,922 INFO [RPCClient-NioEventLoopGroup-6-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testReplayEditsAfterRegionMovedWithMultiCF completed 2024-12-03T23:24:58,922 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testReplayEditsAfterRegionMovedWithMultiCF get assigned. Timeout = 60000ms 2024-12-03T23:24:58,923 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-03T23:24:58,929 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testReplayEditsAfterRegionMovedWithMultiCF assigned to meta. Checking AM states. 2024-12-03T23:24:58,929 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-03T23:24:58,930 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testReplayEditsAfterRegionMovedWithMultiCF assigned. 2024-12-03T23:24:58,943 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1733268298272.1745e7d6d10a6c6039d4c33aadf95a2f., hostname=5f9d64419733,40969,1733268283535, seqNum=2] 2024-12-03T23:24:58,958 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41995 {}] master.HMaster(2410): Client=jenkins//172.17.0.2 move hri=1745e7d6d10a6c6039d4c33aadf95a2f, source=5f9d64419733,40969,1733268283535, destination=5f9d64419733,44215,1733268283642, warming up region on 5f9d64419733,44215,1733268283642 2024-12-03T23:24:58,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41995 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-03T23:24:58,960 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41995 {}] master.HMaster(2414): Client=jenkins//172.17.0.2 move hri=1745e7d6d10a6c6039d4c33aadf95a2f, source=5f9d64419733,40969,1733268283535, destination=5f9d64419733,44215,1733268283642, running balancer 2024-12-03T23:24:58,962 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49547, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-03T23:24:58,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41995 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=1745e7d6d10a6c6039d4c33aadf95a2f, REOPEN/MOVE 2024-12-03T23:24:58,962 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=1745e7d6d10a6c6039d4c33aadf95a2f, REOPEN/MOVE 2024-12-03T23:24:58,964 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=1745e7d6d10a6c6039d4c33aadf95a2f, regionState=CLOSING, regionLocation=5f9d64419733,40969,1733268283535 2024-12-03T23:24:58,966 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=1745e7d6d10a6c6039d4c33aadf95a2f, REOPEN/MOVE because future has completed 2024-12-03T23:24:58,967 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44215 {}] regionserver.RSRpcServices(2066): Warmup testReplayEditsAfterRegionMovedWithMultiCF,,1733268298272.1745e7d6d10a6c6039d4c33aadf95a2f. 2024-12-03T23:24:58,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44215 {}] regionserver.HRegion(7855): Warmup {ENCODED => 1745e7d6d10a6c6039d4c33aadf95a2f, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1733268298272.1745e7d6d10a6c6039d4c33aadf95a2f.', STARTKEY => '', ENDKEY => ''} 2024-12-03T23:24:58,967 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-03T23:24:58,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44215 {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1733268298272.1745e7d6d10a6c6039d4c33aadf95a2f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T23:24:58,967 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure 1745e7d6d10a6c6039d4c33aadf95a2f, server=5f9d64419733,40969,1733268283535}] 2024-12-03T23:24:58,968 INFO [StoreOpener-1745e7d6d10a6c6039d4c33aadf95a2f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 1745e7d6d10a6c6039d4c33aadf95a2f 2024-12-03T23:24:58,969 INFO [StoreOpener-1745e7d6d10a6c6039d4c33aadf95a2f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1745e7d6d10a6c6039d4c33aadf95a2f columnFamilyName cf1 2024-12-03T23:24:58,969 DEBUG [StoreOpener-1745e7d6d10a6c6039d4c33aadf95a2f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T23:24:58,970 INFO [StoreOpener-1745e7d6d10a6c6039d4c33aadf95a2f-1 {}] regionserver.HStore(327): Store=1745e7d6d10a6c6039d4c33aadf95a2f/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T23:24:58,970 INFO [StoreOpener-1745e7d6d10a6c6039d4c33aadf95a2f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 1745e7d6d10a6c6039d4c33aadf95a2f 2024-12-03T23:24:58,971 INFO [StoreOpener-1745e7d6d10a6c6039d4c33aadf95a2f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1745e7d6d10a6c6039d4c33aadf95a2f columnFamilyName cf2 2024-12-03T23:24:58,971 DEBUG [StoreOpener-1745e7d6d10a6c6039d4c33aadf95a2f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T23:24:58,971 INFO [StoreOpener-1745e7d6d10a6c6039d4c33aadf95a2f-1 {}] regionserver.HStore(327): Store=1745e7d6d10a6c6039d4c33aadf95a2f/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T23:24:58,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44215 {}] regionserver.HRegion(1722): Closing 1745e7d6d10a6c6039d4c33aadf95a2f, disabling compactions & flushes 2024-12-03T23:24:58,972 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44215 {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1733268298272.1745e7d6d10a6c6039d4c33aadf95a2f. 2024-12-03T23:24:58,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44215 {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733268298272.1745e7d6d10a6c6039d4c33aadf95a2f. 2024-12-03T23:24:58,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44215 {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733268298272.1745e7d6d10a6c6039d4c33aadf95a2f. after waiting 0 ms 2024-12-03T23:24:58,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44215 {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1733268298272.1745e7d6d10a6c6039d4c33aadf95a2f. 2024-12-03T23:24:58,972 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44215 {}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733268298272.1745e7d6d10a6c6039d4c33aadf95a2f. 2024-12-03T23:24:58,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44215 {}] regionserver.HRegion(1676): Region close journal for 1745e7d6d10a6c6039d4c33aadf95a2f: Waiting for close lock at 1733268298971Disabling compacts and flushes for region at 1733268298972 (+1 ms)Disabling writes for close at 1733268298972Writing region close event to WAL at 1733268298972Closed at 1733268298972 2024-12-03T23:24:59,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41995 {}] procedure.ProcedureSyncWait(219): waitFor pid=7 2024-12-03T23:24:59,125 INFO [RS_CLOSE_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(122): Close 1745e7d6d10a6c6039d4c33aadf95a2f 2024-12-03T23:24:59,125 DEBUG [RS_CLOSE_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-03T23:24:59,126 DEBUG [RS_CLOSE_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1722): Closing 1745e7d6d10a6c6039d4c33aadf95a2f, disabling compactions & flushes 2024-12-03T23:24:59,126 INFO [RS_CLOSE_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1733268298272.1745e7d6d10a6c6039d4c33aadf95a2f. 2024-12-03T23:24:59,126 DEBUG [RS_CLOSE_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733268298272.1745e7d6d10a6c6039d4c33aadf95a2f. 2024-12-03T23:24:59,126 DEBUG [RS_CLOSE_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733268298272.1745e7d6d10a6c6039d4c33aadf95a2f. after waiting 0 ms 2024-12-03T23:24:59,126 DEBUG [RS_CLOSE_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1733268298272.1745e7d6d10a6c6039d4c33aadf95a2f. 2024-12-03T23:24:59,126 INFO [RS_CLOSE_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(2902): Flushing 1745e7d6d10a6c6039d4c33aadf95a2f 2/2 column families, dataSize=31 B heapSize=616 B 2024-12-03T23:24:59,143 DEBUG [RS_CLOSE_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35563/user/jenkins/test-data/e9061e95-7d9e-c12a-b652-9e8a1d87f0cf/data/default/testReplayEditsAfterRegionMovedWithMultiCF/1745e7d6d10a6c6039d4c33aadf95a2f/.tmp/cf1/05b6394056e84f9b800604dce51cd3c9 is 35, key is r1/cf1:q/1733268298945/Put/seqid=0 2024-12-03T23:24:59,150 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39829 is added to blk_1073741887_1065 (size=4783) 2024-12-03T23:24:59,150 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41487 is added to blk_1073741887_1065 (size=4783) 2024-12-03T23:24:59,150 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45491 is added to blk_1073741887_1065 (size=4783) 2024-12-03T23:24:59,151 INFO [RS_CLOSE_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31 B at sequenceid=5 (bloomFilter=false), to=hdfs://localhost:35563/user/jenkins/test-data/e9061e95-7d9e-c12a-b652-9e8a1d87f0cf/data/default/testReplayEditsAfterRegionMovedWithMultiCF/1745e7d6d10a6c6039d4c33aadf95a2f/.tmp/cf1/05b6394056e84f9b800604dce51cd3c9 2024-12-03T23:24:59,158 DEBUG [RS_CLOSE_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35563/user/jenkins/test-data/e9061e95-7d9e-c12a-b652-9e8a1d87f0cf/data/default/testReplayEditsAfterRegionMovedWithMultiCF/1745e7d6d10a6c6039d4c33aadf95a2f/.tmp/cf1/05b6394056e84f9b800604dce51cd3c9 as hdfs://localhost:35563/user/jenkins/test-data/e9061e95-7d9e-c12a-b652-9e8a1d87f0cf/data/default/testReplayEditsAfterRegionMovedWithMultiCF/1745e7d6d10a6c6039d4c33aadf95a2f/cf1/05b6394056e84f9b800604dce51cd3c9 2024-12-03T23:24:59,163 INFO [RS_CLOSE_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35563/user/jenkins/test-data/e9061e95-7d9e-c12a-b652-9e8a1d87f0cf/data/default/testReplayEditsAfterRegionMovedWithMultiCF/1745e7d6d10a6c6039d4c33aadf95a2f/cf1/05b6394056e84f9b800604dce51cd3c9, entries=1, sequenceid=5, filesize=4.7 K 2024-12-03T23:24:59,164 INFO [RS_CLOSE_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~31 B/31, heapSize ~344 B/344, currentSize=0 B/0 for 1745e7d6d10a6c6039d4c33aadf95a2f in 38ms, sequenceid=5, compaction requested=false 2024-12-03T23:24:59,165 DEBUG [RS_CLOSE_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testReplayEditsAfterRegionMovedWithMultiCF' 2024-12-03T23:24:59,170 DEBUG [RS_CLOSE_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35563/user/jenkins/test-data/e9061e95-7d9e-c12a-b652-9e8a1d87f0cf/data/default/testReplayEditsAfterRegionMovedWithMultiCF/1745e7d6d10a6c6039d4c33aadf95a2f/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-12-03T23:24:59,173 INFO [RS_CLOSE_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733268298272.1745e7d6d10a6c6039d4c33aadf95a2f. 2024-12-03T23:24:59,173 DEBUG [RS_CLOSE_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1676): Region close journal for 1745e7d6d10a6c6039d4c33aadf95a2f: Waiting for close lock at 1733268299126Running coprocessor pre-close hooks at 1733268299126Disabling compacts and flushes for region at 1733268299126Disabling writes for close at 1733268299126Obtaining lock to block concurrent updates at 1733268299126Preparing flush snapshotting stores in 1745e7d6d10a6c6039d4c33aadf95a2f at 1733268299126Finished memstore snapshotting testReplayEditsAfterRegionMovedWithMultiCF,,1733268298272.1745e7d6d10a6c6039d4c33aadf95a2f., syncing WAL and waiting on mvcc, flushsize=dataSize=31, getHeapSize=584, getOffHeapSize=0, getCellsCount=1 at 1733268299127 (+1 ms)Flushing stores of testReplayEditsAfterRegionMovedWithMultiCF,,1733268298272.1745e7d6d10a6c6039d4c33aadf95a2f. at 1733268299127Flushing 1745e7d6d10a6c6039d4c33aadf95a2f/cf1: creating writer at 1733268299128 (+1 ms)Flushing 1745e7d6d10a6c6039d4c33aadf95a2f/cf1: appending metadata at 1733268299143 (+15 ms)Flushing 1745e7d6d10a6c6039d4c33aadf95a2f/cf1: closing flushed file at 1733268299143Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@18c1ee95: reopening flushed file at 1733268299157 (+14 ms)Finished flush of dataSize ~31 B/31, heapSize ~344 B/344, currentSize=0 B/0 for 1745e7d6d10a6c6039d4c33aadf95a2f in 38ms, sequenceid=5, compaction requested=false at 1733268299165 (+8 ms)Writing region close event to WAL at 1733268299166 (+1 ms)Running coprocessor post-close hooks at 1733268299170 (+4 ms)Closed at 1733268299173 (+3 ms) 2024-12-03T23:24:59,173 INFO [RS_CLOSE_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionServer(3302): Adding 1745e7d6d10a6c6039d4c33aadf95a2f move to 5f9d64419733,44215,1733268283642 record at close sequenceid=5 2024-12-03T23:24:59,176 INFO [RS_CLOSE_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(157): Closed 1745e7d6d10a6c6039d4c33aadf95a2f 2024-12-03T23:24:59,177 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=1745e7d6d10a6c6039d4c33aadf95a2f, regionState=CLOSED 2024-12-03T23:24:59,180 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure 1745e7d6d10a6c6039d4c33aadf95a2f, server=5f9d64419733,40969,1733268283535 because future has completed 2024-12-03T23:24:59,185 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-12-03T23:24:59,185 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; CloseRegionProcedure 1745e7d6d10a6c6039d4c33aadf95a2f, server=5f9d64419733,40969,1733268283535 in 214 msec 2024-12-03T23:24:59,186 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=1745e7d6d10a6c6039d4c33aadf95a2f, REOPEN/MOVE; state=CLOSED, location=5f9d64419733,44215,1733268283642; forceNewPlan=false, retain=false 2024-12-03T23:24:59,336 INFO [5f9d64419733:41995 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-12-03T23:24:59,337 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=1745e7d6d10a6c6039d4c33aadf95a2f, regionState=OPENING, regionLocation=5f9d64419733,44215,1733268283642 2024-12-03T23:24:59,342 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=1745e7d6d10a6c6039d4c33aadf95a2f, REOPEN/MOVE because future has completed 2024-12-03T23:24:59,343 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1745e7d6d10a6c6039d4c33aadf95a2f, server=5f9d64419733,44215,1733268283642}] 2024-12-03T23:24:59,501 INFO [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1733268298272.1745e7d6d10a6c6039d4c33aadf95a2f. 2024-12-03T23:24:59,501 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => 1745e7d6d10a6c6039d4c33aadf95a2f, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1733268298272.1745e7d6d10a6c6039d4c33aadf95a2f.', STARTKEY => '', ENDKEY => ''} 2024-12-03T23:24:59,501 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF 1745e7d6d10a6c6039d4c33aadf95a2f 2024-12-03T23:24:59,501 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1733268298272.1745e7d6d10a6c6039d4c33aadf95a2f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T23:24:59,502 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for 1745e7d6d10a6c6039d4c33aadf95a2f 2024-12-03T23:24:59,502 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for 1745e7d6d10a6c6039d4c33aadf95a2f 2024-12-03T23:24:59,503 INFO [StoreOpener-1745e7d6d10a6c6039d4c33aadf95a2f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 1745e7d6d10a6c6039d4c33aadf95a2f 2024-12-03T23:24:59,504 INFO [StoreOpener-1745e7d6d10a6c6039d4c33aadf95a2f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1745e7d6d10a6c6039d4c33aadf95a2f columnFamilyName cf1 2024-12-03T23:24:59,504 DEBUG [StoreOpener-1745e7d6d10a6c6039d4c33aadf95a2f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T23:24:59,511 DEBUG [StoreOpener-1745e7d6d10a6c6039d4c33aadf95a2f-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:35563/user/jenkins/test-data/e9061e95-7d9e-c12a-b652-9e8a1d87f0cf/data/default/testReplayEditsAfterRegionMovedWithMultiCF/1745e7d6d10a6c6039d4c33aadf95a2f/cf1/05b6394056e84f9b800604dce51cd3c9 2024-12-03T23:24:59,511 INFO [StoreOpener-1745e7d6d10a6c6039d4c33aadf95a2f-1 {}] regionserver.HStore(327): Store=1745e7d6d10a6c6039d4c33aadf95a2f/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T23:24:59,512 INFO [StoreOpener-1745e7d6d10a6c6039d4c33aadf95a2f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 1745e7d6d10a6c6039d4c33aadf95a2f 2024-12-03T23:24:59,512 INFO [StoreOpener-1745e7d6d10a6c6039d4c33aadf95a2f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1745e7d6d10a6c6039d4c33aadf95a2f columnFamilyName cf2 2024-12-03T23:24:59,513 DEBUG [StoreOpener-1745e7d6d10a6c6039d4c33aadf95a2f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T23:24:59,513 INFO [StoreOpener-1745e7d6d10a6c6039d4c33aadf95a2f-1 {}] regionserver.HStore(327): Store=1745e7d6d10a6c6039d4c33aadf95a2f/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T23:24:59,513 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for 1745e7d6d10a6c6039d4c33aadf95a2f 2024-12-03T23:24:59,514 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35563/user/jenkins/test-data/e9061e95-7d9e-c12a-b652-9e8a1d87f0cf/data/default/testReplayEditsAfterRegionMovedWithMultiCF/1745e7d6d10a6c6039d4c33aadf95a2f 2024-12-03T23:24:59,515 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35563/user/jenkins/test-data/e9061e95-7d9e-c12a-b652-9e8a1d87f0cf/data/default/testReplayEditsAfterRegionMovedWithMultiCF/1745e7d6d10a6c6039d4c33aadf95a2f 2024-12-03T23:24:59,516 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for 1745e7d6d10a6c6039d4c33aadf95a2f 2024-12-03T23:24:59,516 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for 1745e7d6d10a6c6039d4c33aadf95a2f 2024-12-03T23:24:59,517 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-12-03T23:24:59,518 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for 1745e7d6d10a6c6039d4c33aadf95a2f 2024-12-03T23:24:59,519 INFO [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1114): Opened 1745e7d6d10a6c6039d4c33aadf95a2f; next sequenceid=9; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67708430, jitterRate=0.008934229612350464}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-12-03T23:24:59,519 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1745e7d6d10a6c6039d4c33aadf95a2f 2024-12-03T23:24:59,520 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for 1745e7d6d10a6c6039d4c33aadf95a2f: Running coprocessor pre-open hook at 1733268299502Writing region info on filesystem at 1733268299502Initializing all the Stores at 1733268299503 (+1 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733268299503Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733268299503Cleaning up temporary data from old regions at 1733268299516 (+13 ms)Running coprocessor post-open hooks at 1733268299519 (+3 ms)Region opened successfully at 1733268299520 (+1 ms) 2024-12-03T23:24:59,521 INFO [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1733268298272.1745e7d6d10a6c6039d4c33aadf95a2f., pid=9, masterSystemTime=1733268299496 2024-12-03T23:24:59,523 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1733268298272.1745e7d6d10a6c6039d4c33aadf95a2f. 2024-12-03T23:24:59,523 INFO [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1733268298272.1745e7d6d10a6c6039d4c33aadf95a2f. 2024-12-03T23:24:59,524 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=1745e7d6d10a6c6039d4c33aadf95a2f, regionState=OPEN, openSeqNum=9, regionLocation=5f9d64419733,44215,1733268283642 2024-12-03T23:24:59,526 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1745e7d6d10a6c6039d4c33aadf95a2f, server=5f9d64419733,44215,1733268283642 because future has completed 2024-12-03T23:24:59,530 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=7 2024-12-03T23:24:59,530 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=7, state=SUCCESS, hasLock=false; OpenRegionProcedure 1745e7d6d10a6c6039d4c33aadf95a2f, server=5f9d64419733,44215,1733268283642 in 185 msec 2024-12-03T23:24:59,532 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=1745e7d6d10a6c6039d4c33aadf95a2f, REOPEN/MOVE in 570 msec 2024-12-03T23:24:59,569 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-03T23:24:59,571 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41886, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-03T23:24:59,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40969 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 139 connection: 172.17.0.2:34998 deadline: 1733268359575, exception=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=5f9d64419733 port=44215 startCode=1733268283642. As of locationSeqNum=5. 2024-12-03T23:24:59,602 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733268298272.1745e7d6d10a6c6039d4c33aadf95a2f., hostname=5f9d64419733,40969,1733268283535, seqNum=2 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1733268298272.1745e7d6d10a6c6039d4c33aadf95a2f., hostname=5f9d64419733,40969,1733268283535, seqNum=2, error=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=5f9d64419733 port=44215 startCode=1733268283642. As of locationSeqNum=5. 2024-12-03T23:24:59,603 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733268298272.1745e7d6d10a6c6039d4c33aadf95a2f., hostname=5f9d64419733,40969,1733268283535, seqNum=2 is org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=5f9d64419733 port=44215 startCode=1733268283642. As of locationSeqNum=5. 2024-12-03T23:24:59,603 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(84): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733268298272.1745e7d6d10a6c6039d4c33aadf95a2f., hostname=5f9d64419733,40969,1733268283535, seqNum=2 with the new location region=testReplayEditsAfterRegionMovedWithMultiCF,,1733268298272.1745e7d6d10a6c6039d4c33aadf95a2f., hostname=5f9d64419733,44215,1733268283642, seqNum=5 constructed by org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=5f9d64419733 port=44215 startCode=1733268283642. As of locationSeqNum=5. 2024-12-03T23:24:59,711 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T23:24:59,713 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41894, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T23:24:59,724 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 1745e7d6d10a6c6039d4c33aadf95a2f 2/2 column families, dataSize=50 B heapSize=720 B 2024-12-03T23:24:59,740 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35563/user/jenkins/test-data/e9061e95-7d9e-c12a-b652-9e8a1d87f0cf/data/default/testReplayEditsAfterRegionMovedWithMultiCF/1745e7d6d10a6c6039d4c33aadf95a2f/.tmp/cf1/e115ce79380448cbbe436f379baa6f7a is 29, key is r1/cf1:/1733268299715/DeleteFamily/seqid=0 2024-12-03T23:24:59,750 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45491 is added to blk_1073741888_1066 (size=4906) 2024-12-03T23:24:59,750 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39829 is added to blk_1073741888_1066 (size=4906) 2024-12-03T23:24:59,751 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41487 is added to blk_1073741888_1066 (size=4906) 2024-12-03T23:24:59,751 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=25 B at sequenceid=12 (bloomFilter=false), to=hdfs://localhost:35563/user/jenkins/test-data/e9061e95-7d9e-c12a-b652-9e8a1d87f0cf/data/default/testReplayEditsAfterRegionMovedWithMultiCF/1745e7d6d10a6c6039d4c33aadf95a2f/.tmp/cf1/e115ce79380448cbbe436f379baa6f7a 2024-12-03T23:24:59,758 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for e115ce79380448cbbe436f379baa6f7a 2024-12-03T23:24:59,774 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35563/user/jenkins/test-data/e9061e95-7d9e-c12a-b652-9e8a1d87f0cf/data/default/testReplayEditsAfterRegionMovedWithMultiCF/1745e7d6d10a6c6039d4c33aadf95a2f/.tmp/cf2/a6581cdd804c48c291109806c75d3e85 is 29, key is r1/cf2:/1733268299715/DeleteFamily/seqid=0 2024-12-03T23:24:59,781 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41487 is added to blk_1073741889_1067 (size=4906) 2024-12-03T23:24:59,781 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39829 is added to blk_1073741889_1067 (size=4906) 2024-12-03T23:24:59,782 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45491 is added to blk_1073741889_1067 (size=4906) 2024-12-03T23:24:59,782 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=25 B at sequenceid=12 (bloomFilter=false), to=hdfs://localhost:35563/user/jenkins/test-data/e9061e95-7d9e-c12a-b652-9e8a1d87f0cf/data/default/testReplayEditsAfterRegionMovedWithMultiCF/1745e7d6d10a6c6039d4c33aadf95a2f/.tmp/cf2/a6581cdd804c48c291109806c75d3e85 2024-12-03T23:24:59,788 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for a6581cdd804c48c291109806c75d3e85 2024-12-03T23:24:59,789 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35563/user/jenkins/test-data/e9061e95-7d9e-c12a-b652-9e8a1d87f0cf/data/default/testReplayEditsAfterRegionMovedWithMultiCF/1745e7d6d10a6c6039d4c33aadf95a2f/.tmp/cf1/e115ce79380448cbbe436f379baa6f7a as hdfs://localhost:35563/user/jenkins/test-data/e9061e95-7d9e-c12a-b652-9e8a1d87f0cf/data/default/testReplayEditsAfterRegionMovedWithMultiCF/1745e7d6d10a6c6039d4c33aadf95a2f/cf1/e115ce79380448cbbe436f379baa6f7a 2024-12-03T23:24:59,795 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for e115ce79380448cbbe436f379baa6f7a 2024-12-03T23:24:59,795 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35563/user/jenkins/test-data/e9061e95-7d9e-c12a-b652-9e8a1d87f0cf/data/default/testReplayEditsAfterRegionMovedWithMultiCF/1745e7d6d10a6c6039d4c33aadf95a2f/cf1/e115ce79380448cbbe436f379baa6f7a, entries=1, sequenceid=12, filesize=4.8 K 2024-12-03T23:24:59,796 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35563/user/jenkins/test-data/e9061e95-7d9e-c12a-b652-9e8a1d87f0cf/data/default/testReplayEditsAfterRegionMovedWithMultiCF/1745e7d6d10a6c6039d4c33aadf95a2f/.tmp/cf2/a6581cdd804c48c291109806c75d3e85 as hdfs://localhost:35563/user/jenkins/test-data/e9061e95-7d9e-c12a-b652-9e8a1d87f0cf/data/default/testReplayEditsAfterRegionMovedWithMultiCF/1745e7d6d10a6c6039d4c33aadf95a2f/cf2/a6581cdd804c48c291109806c75d3e85 2024-12-03T23:24:59,802 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for a6581cdd804c48c291109806c75d3e85 2024-12-03T23:24:59,802 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35563/user/jenkins/test-data/e9061e95-7d9e-c12a-b652-9e8a1d87f0cf/data/default/testReplayEditsAfterRegionMovedWithMultiCF/1745e7d6d10a6c6039d4c33aadf95a2f/cf2/a6581cdd804c48c291109806c75d3e85, entries=1, sequenceid=12, filesize=4.8 K 2024-12-03T23:24:59,803 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~50 B/50, heapSize ~688 B/688, currentSize=0 B/0 for 1745e7d6d10a6c6039d4c33aadf95a2f in 79ms, sequenceid=12, compaction requested=false 2024-12-03T23:24:59,804 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 1745e7d6d10a6c6039d4c33aadf95a2f: 2024-12-03T23:24:59,807 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-12-03T23:24:59,809 DEBUG [Time-limited test {}] regionserver.HStore(1541): 1745e7d6d10a6c6039d4c33aadf95a2f/cf1 is initiating major compaction (all files) 2024-12-03T23:24:59,810 INFO [Time-limited test {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-03T23:24:59,810 INFO [Time-limited test {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-03T23:24:59,810 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of 1745e7d6d10a6c6039d4c33aadf95a2f/cf1 in testReplayEditsAfterRegionMovedWithMultiCF,,1733268298272.1745e7d6d10a6c6039d4c33aadf95a2f. 2024-12-03T23:24:59,811 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35563/user/jenkins/test-data/e9061e95-7d9e-c12a-b652-9e8a1d87f0cf/data/default/testReplayEditsAfterRegionMovedWithMultiCF/1745e7d6d10a6c6039d4c33aadf95a2f/cf1/05b6394056e84f9b800604dce51cd3c9, hdfs://localhost:35563/user/jenkins/test-data/e9061e95-7d9e-c12a-b652-9e8a1d87f0cf/data/default/testReplayEditsAfterRegionMovedWithMultiCF/1745e7d6d10a6c6039d4c33aadf95a2f/cf1/e115ce79380448cbbe436f379baa6f7a] into tmpdir=hdfs://localhost:35563/user/jenkins/test-data/e9061e95-7d9e-c12a-b652-9e8a1d87f0cf/data/default/testReplayEditsAfterRegionMovedWithMultiCF/1745e7d6d10a6c6039d4c33aadf95a2f/.tmp, totalSize=9.5 K 2024-12-03T23:24:59,812 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 05b6394056e84f9b800604dce51cd3c9, keycount=1, bloomtype=NONE, size=4.7 K, encoding=NONE, compression=NONE, seqNum=5, earliestPutTs=1733268298945 2024-12-03T23:24:59,812 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting e115ce79380448cbbe436f379baa6f7a, keycount=1, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=9223372036854775807 2024-12-03T23:24:59,826 INFO [Time-limited test {}] throttle.PressureAwareThroughputController(145): 1745e7d6d10a6c6039d4c33aadf95a2f#cf1#compaction#16 average throughput is 0.00 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T23:24:59,832 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45491 is added to blk_1073741890_1068 (size=4626) 2024-12-03T23:24:59,833 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39829 is added to blk_1073741890_1068 (size=4626) 2024-12-03T23:24:59,833 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41487 is added to blk_1073741890_1068 (size=4626) 2024-12-03T23:24:59,840 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35563/user/jenkins/test-data/e9061e95-7d9e-c12a-b652-9e8a1d87f0cf/data/default/testReplayEditsAfterRegionMovedWithMultiCF/1745e7d6d10a6c6039d4c33aadf95a2f/.tmp/cf1/7d744d5820004695a930452e689f1b2a as hdfs://localhost:35563/user/jenkins/test-data/e9061e95-7d9e-c12a-b652-9e8a1d87f0cf/data/default/testReplayEditsAfterRegionMovedWithMultiCF/1745e7d6d10a6c6039d4c33aadf95a2f/cf1/7d744d5820004695a930452e689f1b2a 2024-12-03T23:24:59,854 INFO [Time-limited test {}] regionserver.HStore(1337): Completed major compaction of 2 (all) file(s) in 1745e7d6d10a6c6039d4c33aadf95a2f/cf1 of 1745e7d6d10a6c6039d4c33aadf95a2f into 7d744d5820004695a930452e689f1b2a(size=4.5 K), total size for store is 4.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T23:24:59,855 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for 1745e7d6d10a6c6039d4c33aadf95a2f: 2024-12-03T23:24:59,855 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 1 store files, 0 compacting, 1 eligible, 16 blocking 2024-12-03T23:24:59,855 DEBUG [Time-limited test {}] regionserver.HStore(1541): 1745e7d6d10a6c6039d4c33aadf95a2f/cf2 is initiating major compaction (all files) 2024-12-03T23:24:59,855 INFO [Time-limited test {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-03T23:24:59,855 INFO [Time-limited test {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-03T23:24:59,855 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of 1745e7d6d10a6c6039d4c33aadf95a2f/cf2 in testReplayEditsAfterRegionMovedWithMultiCF,,1733268298272.1745e7d6d10a6c6039d4c33aadf95a2f. 2024-12-03T23:24:59,855 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35563/user/jenkins/test-data/e9061e95-7d9e-c12a-b652-9e8a1d87f0cf/data/default/testReplayEditsAfterRegionMovedWithMultiCF/1745e7d6d10a6c6039d4c33aadf95a2f/cf2/a6581cdd804c48c291109806c75d3e85] into tmpdir=hdfs://localhost:35563/user/jenkins/test-data/e9061e95-7d9e-c12a-b652-9e8a1d87f0cf/data/default/testReplayEditsAfterRegionMovedWithMultiCF/1745e7d6d10a6c6039d4c33aadf95a2f/.tmp, totalSize=4.8 K 2024-12-03T23:24:59,856 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting a6581cdd804c48c291109806c75d3e85, keycount=1, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=9223372036854775807 2024-12-03T23:24:59,863 INFO [Time-limited test {}] throttle.PressureAwareThroughputController(145): 1745e7d6d10a6c6039d4c33aadf95a2f#cf2#compaction#17 average throughput is NaN MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T23:24:59,871 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45491 is added to blk_1073741891_1069 (size=4592) 2024-12-03T23:24:59,872 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41487 is added to blk_1073741891_1069 (size=4592) 2024-12-03T23:24:59,872 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39829 is added to blk_1073741891_1069 (size=4592) 2024-12-03T23:24:59,881 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35563/user/jenkins/test-data/e9061e95-7d9e-c12a-b652-9e8a1d87f0cf/data/default/testReplayEditsAfterRegionMovedWithMultiCF/1745e7d6d10a6c6039d4c33aadf95a2f/.tmp/cf2/fa67e9af7de2471d97906c12a08b2665 as hdfs://localhost:35563/user/jenkins/test-data/e9061e95-7d9e-c12a-b652-9e8a1d87f0cf/data/default/testReplayEditsAfterRegionMovedWithMultiCF/1745e7d6d10a6c6039d4c33aadf95a2f/cf2/fa67e9af7de2471d97906c12a08b2665 2024-12-03T23:24:59,887 INFO [Time-limited test {}] regionserver.HStore(1337): Completed major compaction of 1 (all) file(s) in 1745e7d6d10a6c6039d4c33aadf95a2f/cf2 of 1745e7d6d10a6c6039d4c33aadf95a2f into fa67e9af7de2471d97906c12a08b2665(size=4.5 K), total size for store is 4.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T23:24:59,887 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for 1745e7d6d10a6c6039d4c33aadf95a2f: 2024-12-03T23:24:59,892 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41995 {}] master.HMaster(2410): Client=jenkins//172.17.0.2 move hri=1745e7d6d10a6c6039d4c33aadf95a2f, source=5f9d64419733,44215,1733268283642, destination=5f9d64419733,40969,1733268283535, warming up region on 5f9d64419733,40969,1733268283535 2024-12-03T23:24:59,892 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41995 {}] master.HMaster(2414): Client=jenkins//172.17.0.2 move hri=1745e7d6d10a6c6039d4c33aadf95a2f, source=5f9d64419733,44215,1733268283642, destination=5f9d64419733,40969,1733268283535, running balancer 2024-12-03T23:24:59,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41995 {}] procedure2.ProcedureExecutor(1139): Stored pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=1745e7d6d10a6c6039d4c33aadf95a2f, REOPEN/MOVE 2024-12-03T23:24:59,894 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=1745e7d6d10a6c6039d4c33aadf95a2f, REOPEN/MOVE 2024-12-03T23:24:59,896 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=1745e7d6d10a6c6039d4c33aadf95a2f, regionState=CLOSING, regionLocation=5f9d64419733,44215,1733268283642 2024-12-03T23:24:59,896 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40969 {}] regionserver.RSRpcServices(2066): Warmup testReplayEditsAfterRegionMovedWithMultiCF,,1733268298272.1745e7d6d10a6c6039d4c33aadf95a2f. 2024-12-03T23:24:59,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40969 {}] regionserver.HRegion(7855): Warmup {ENCODED => 1745e7d6d10a6c6039d4c33aadf95a2f, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1733268298272.1745e7d6d10a6c6039d4c33aadf95a2f.', STARTKEY => '', ENDKEY => ''} 2024-12-03T23:24:59,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40969 {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1733268298272.1745e7d6d10a6c6039d4c33aadf95a2f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T23:24:59,897 INFO [StoreOpener-1745e7d6d10a6c6039d4c33aadf95a2f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 1745e7d6d10a6c6039d4c33aadf95a2f 2024-12-03T23:24:59,898 INFO [StoreOpener-1745e7d6d10a6c6039d4c33aadf95a2f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1745e7d6d10a6c6039d4c33aadf95a2f columnFamilyName cf1 2024-12-03T23:24:59,898 DEBUG [StoreOpener-1745e7d6d10a6c6039d4c33aadf95a2f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T23:24:59,899 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=1745e7d6d10a6c6039d4c33aadf95a2f, REOPEN/MOVE because future has completed 2024-12-03T23:24:59,899 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-03T23:24:59,899 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=11, ppid=10, state=RUNNABLE, hasLock=false; CloseRegionProcedure 1745e7d6d10a6c6039d4c33aadf95a2f, server=5f9d64419733,44215,1733268283642}] 2024-12-03T23:24:59,906 DEBUG [StoreOpener-1745e7d6d10a6c6039d4c33aadf95a2f-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:35563/user/jenkins/test-data/e9061e95-7d9e-c12a-b652-9e8a1d87f0cf/data/default/testReplayEditsAfterRegionMovedWithMultiCF/1745e7d6d10a6c6039d4c33aadf95a2f/cf1/05b6394056e84f9b800604dce51cd3c9 2024-12-03T23:24:59,910 DEBUG [StoreOpener-1745e7d6d10a6c6039d4c33aadf95a2f-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:35563/user/jenkins/test-data/e9061e95-7d9e-c12a-b652-9e8a1d87f0cf/data/default/testReplayEditsAfterRegionMovedWithMultiCF/1745e7d6d10a6c6039d4c33aadf95a2f/cf1/7d744d5820004695a930452e689f1b2a 2024-12-03T23:24:59,915 INFO [StoreFileOpener-1745e7d6d10a6c6039d4c33aadf95a2f-cf1-1 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for e115ce79380448cbbe436f379baa6f7a 2024-12-03T23:24:59,915 DEBUG [StoreOpener-1745e7d6d10a6c6039d4c33aadf95a2f-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:35563/user/jenkins/test-data/e9061e95-7d9e-c12a-b652-9e8a1d87f0cf/data/default/testReplayEditsAfterRegionMovedWithMultiCF/1745e7d6d10a6c6039d4c33aadf95a2f/cf1/e115ce79380448cbbe436f379baa6f7a 2024-12-03T23:24:59,915 INFO [StoreOpener-1745e7d6d10a6c6039d4c33aadf95a2f-1 {}] regionserver.HStore(327): Store=1745e7d6d10a6c6039d4c33aadf95a2f/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T23:24:59,915 INFO [StoreOpener-1745e7d6d10a6c6039d4c33aadf95a2f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 1745e7d6d10a6c6039d4c33aadf95a2f 2024-12-03T23:24:59,916 INFO [StoreOpener-1745e7d6d10a6c6039d4c33aadf95a2f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1745e7d6d10a6c6039d4c33aadf95a2f columnFamilyName cf2 2024-12-03T23:24:59,916 DEBUG [StoreOpener-1745e7d6d10a6c6039d4c33aadf95a2f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T23:24:59,922 INFO [StoreFileOpener-1745e7d6d10a6c6039d4c33aadf95a2f-cf2-1 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for a6581cdd804c48c291109806c75d3e85 2024-12-03T23:24:59,922 DEBUG [StoreOpener-1745e7d6d10a6c6039d4c33aadf95a2f-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:35563/user/jenkins/test-data/e9061e95-7d9e-c12a-b652-9e8a1d87f0cf/data/default/testReplayEditsAfterRegionMovedWithMultiCF/1745e7d6d10a6c6039d4c33aadf95a2f/cf2/a6581cdd804c48c291109806c75d3e85 2024-12-03T23:24:59,926 DEBUG [StoreOpener-1745e7d6d10a6c6039d4c33aadf95a2f-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:35563/user/jenkins/test-data/e9061e95-7d9e-c12a-b652-9e8a1d87f0cf/data/default/testReplayEditsAfterRegionMovedWithMultiCF/1745e7d6d10a6c6039d4c33aadf95a2f/cf2/fa67e9af7de2471d97906c12a08b2665 2024-12-03T23:24:59,926 INFO [StoreOpener-1745e7d6d10a6c6039d4c33aadf95a2f-1 {}] regionserver.HStore(327): Store=1745e7d6d10a6c6039d4c33aadf95a2f/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T23:24:59,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40969 {}] regionserver.HRegion(1722): Closing 1745e7d6d10a6c6039d4c33aadf95a2f, disabling compactions & flushes 2024-12-03T23:24:59,927 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40969 {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1733268298272.1745e7d6d10a6c6039d4c33aadf95a2f. 2024-12-03T23:24:59,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40969 {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733268298272.1745e7d6d10a6c6039d4c33aadf95a2f. 2024-12-03T23:24:59,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40969 {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733268298272.1745e7d6d10a6c6039d4c33aadf95a2f. after waiting 0 ms 2024-12-03T23:24:59,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40969 {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1733268298272.1745e7d6d10a6c6039d4c33aadf95a2f. 2024-12-03T23:24:59,928 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40969 {}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733268298272.1745e7d6d10a6c6039d4c33aadf95a2f. 2024-12-03T23:24:59,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40969 {}] regionserver.HRegion(1676): Region close journal for 1745e7d6d10a6c6039d4c33aadf95a2f: Waiting for close lock at 1733268299927Disabling compacts and flushes for region at 1733268299927Disabling writes for close at 1733268299927Writing region close event to WAL at 1733268299928 (+1 ms)Closed at 1733268299928 2024-12-03T23:24:59,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41995 {}] procedure.ProcedureSyncWait(219): waitFor pid=10 2024-12-03T23:25:00,054 INFO [RS_CLOSE_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] handler.UnassignRegionHandler(122): Close 1745e7d6d10a6c6039d4c33aadf95a2f 2024-12-03T23:25:00,054 DEBUG [RS_CLOSE_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-03T23:25:00,054 DEBUG [RS_CLOSE_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1722): Closing 1745e7d6d10a6c6039d4c33aadf95a2f, disabling compactions & flushes 2024-12-03T23:25:00,054 INFO [RS_CLOSE_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1733268298272.1745e7d6d10a6c6039d4c33aadf95a2f. 2024-12-03T23:25:00,054 DEBUG [RS_CLOSE_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733268298272.1745e7d6d10a6c6039d4c33aadf95a2f. 2024-12-03T23:25:00,054 DEBUG [RS_CLOSE_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733268298272.1745e7d6d10a6c6039d4c33aadf95a2f. after waiting 0 ms 2024-12-03T23:25:00,054 DEBUG [RS_CLOSE_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1733268298272.1745e7d6d10a6c6039d4c33aadf95a2f. 2024-12-03T23:25:00,054 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1733268298272.1745e7d6d10a6c6039d4c33aadf95a2f.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35563/user/jenkins/test-data/e9061e95-7d9e-c12a-b652-9e8a1d87f0cf/data/default/testReplayEditsAfterRegionMovedWithMultiCF/1745e7d6d10a6c6039d4c33aadf95a2f/cf1/05b6394056e84f9b800604dce51cd3c9, hdfs://localhost:35563/user/jenkins/test-data/e9061e95-7d9e-c12a-b652-9e8a1d87f0cf/data/default/testReplayEditsAfterRegionMovedWithMultiCF/1745e7d6d10a6c6039d4c33aadf95a2f/cf1/e115ce79380448cbbe436f379baa6f7a] to archive 2024-12-03T23:25:00,058 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1733268298272.1745e7d6d10a6c6039d4c33aadf95a2f.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-03T23:25:00,061 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1733268298272.1745e7d6d10a6c6039d4c33aadf95a2f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35563/user/jenkins/test-data/e9061e95-7d9e-c12a-b652-9e8a1d87f0cf/data/default/testReplayEditsAfterRegionMovedWithMultiCF/1745e7d6d10a6c6039d4c33aadf95a2f/cf1/05b6394056e84f9b800604dce51cd3c9 to hdfs://localhost:35563/user/jenkins/test-data/e9061e95-7d9e-c12a-b652-9e8a1d87f0cf/archive/data/default/testReplayEditsAfterRegionMovedWithMultiCF/1745e7d6d10a6c6039d4c33aadf95a2f/cf1/05b6394056e84f9b800604dce51cd3c9 2024-12-03T23:25:00,063 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1733268298272.1745e7d6d10a6c6039d4c33aadf95a2f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35563/user/jenkins/test-data/e9061e95-7d9e-c12a-b652-9e8a1d87f0cf/data/default/testReplayEditsAfterRegionMovedWithMultiCF/1745e7d6d10a6c6039d4c33aadf95a2f/cf1/e115ce79380448cbbe436f379baa6f7a to hdfs://localhost:35563/user/jenkins/test-data/e9061e95-7d9e-c12a-b652-9e8a1d87f0cf/archive/data/default/testReplayEditsAfterRegionMovedWithMultiCF/1745e7d6d10a6c6039d4c33aadf95a2f/cf1/e115ce79380448cbbe436f379baa6f7a 2024-12-03T23:25:00,079 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1733268298272.1745e7d6d10a6c6039d4c33aadf95a2f.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35563/user/jenkins/test-data/e9061e95-7d9e-c12a-b652-9e8a1d87f0cf/data/default/testReplayEditsAfterRegionMovedWithMultiCF/1745e7d6d10a6c6039d4c33aadf95a2f/cf2/a6581cdd804c48c291109806c75d3e85] to archive 2024-12-03T23:25:00,080 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1733268298272.1745e7d6d10a6c6039d4c33aadf95a2f.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-03T23:25:00,082 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1733268298272.1745e7d6d10a6c6039d4c33aadf95a2f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35563/user/jenkins/test-data/e9061e95-7d9e-c12a-b652-9e8a1d87f0cf/data/default/testReplayEditsAfterRegionMovedWithMultiCF/1745e7d6d10a6c6039d4c33aadf95a2f/cf2/a6581cdd804c48c291109806c75d3e85 to hdfs://localhost:35563/user/jenkins/test-data/e9061e95-7d9e-c12a-b652-9e8a1d87f0cf/archive/data/default/testReplayEditsAfterRegionMovedWithMultiCF/1745e7d6d10a6c6039d4c33aadf95a2f/cf2/a6581cdd804c48c291109806c75d3e85 2024-12-03T23:25:00,087 DEBUG [RS_CLOSE_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35563/user/jenkins/test-data/e9061e95-7d9e-c12a-b652-9e8a1d87f0cf/data/default/testReplayEditsAfterRegionMovedWithMultiCF/1745e7d6d10a6c6039d4c33aadf95a2f/recovered.edits/17.seqid, newMaxSeqId=17, maxSeqId=8 2024-12-03T23:25:00,088 INFO [RS_CLOSE_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733268298272.1745e7d6d10a6c6039d4c33aadf95a2f. 2024-12-03T23:25:00,088 DEBUG [RS_CLOSE_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1676): Region close journal for 1745e7d6d10a6c6039d4c33aadf95a2f: Waiting for close lock at 1733268300054Running coprocessor pre-close hooks at 1733268300054Disabling compacts and flushes for region at 1733268300054Disabling writes for close at 1733268300054Writing region close event to WAL at 1733268300083 (+29 ms)Running coprocessor post-close hooks at 1733268300088 (+5 ms)Closed at 1733268300088 2024-12-03T23:25:00,088 INFO [RS_CLOSE_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegionServer(3302): Adding 1745e7d6d10a6c6039d4c33aadf95a2f move to 5f9d64419733,40969,1733268283535 record at close sequenceid=12 2024-12-03T23:25:00,091 INFO [RS_CLOSE_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] handler.UnassignRegionHandler(157): Closed 1745e7d6d10a6c6039d4c33aadf95a2f 2024-12-03T23:25:00,092 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=1745e7d6d10a6c6039d4c33aadf95a2f, regionState=CLOSED 2024-12-03T23:25:00,095 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=10, state=RUNNABLE, hasLock=false; CloseRegionProcedure 1745e7d6d10a6c6039d4c33aadf95a2f, server=5f9d64419733,44215,1733268283642 because future has completed 2024-12-03T23:25:00,099 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=11, resume processing ppid=10 2024-12-03T23:25:00,099 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=10, state=SUCCESS, hasLock=false; CloseRegionProcedure 1745e7d6d10a6c6039d4c33aadf95a2f, server=5f9d64419733,44215,1733268283642 in 197 msec 2024-12-03T23:25:00,100 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=1745e7d6d10a6c6039d4c33aadf95a2f, REOPEN/MOVE; state=CLOSED, location=5f9d64419733,40969,1733268283535; forceNewPlan=false, retain=false 2024-12-03T23:25:00,251 INFO [5f9d64419733:41995 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-12-03T23:25:00,251 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=1745e7d6d10a6c6039d4c33aadf95a2f, regionState=OPENING, regionLocation=5f9d64419733,40969,1733268283535 2024-12-03T23:25:00,255 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=1745e7d6d10a6c6039d4c33aadf95a2f, REOPEN/MOVE because future has completed 2024-12-03T23:25:00,255 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1745e7d6d10a6c6039d4c33aadf95a2f, server=5f9d64419733,40969,1733268283535}] 2024-12-03T23:25:00,412 INFO [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1733268298272.1745e7d6d10a6c6039d4c33aadf95a2f. 2024-12-03T23:25:00,412 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7752): Opening region: {ENCODED => 1745e7d6d10a6c6039d4c33aadf95a2f, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1733268298272.1745e7d6d10a6c6039d4c33aadf95a2f.', STARTKEY => '', ENDKEY => ''} 2024-12-03T23:25:00,413 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF 1745e7d6d10a6c6039d4c33aadf95a2f 2024-12-03T23:25:00,413 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1733268298272.1745e7d6d10a6c6039d4c33aadf95a2f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T23:25:00,413 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7794): checking encryption for 1745e7d6d10a6c6039d4c33aadf95a2f 2024-12-03T23:25:00,413 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7797): checking classloading for 1745e7d6d10a6c6039d4c33aadf95a2f 2024-12-03T23:25:00,414 INFO [StoreOpener-1745e7d6d10a6c6039d4c33aadf95a2f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 1745e7d6d10a6c6039d4c33aadf95a2f 2024-12-03T23:25:00,415 INFO [StoreOpener-1745e7d6d10a6c6039d4c33aadf95a2f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1745e7d6d10a6c6039d4c33aadf95a2f columnFamilyName cf1 2024-12-03T23:25:00,416 DEBUG [StoreOpener-1745e7d6d10a6c6039d4c33aadf95a2f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T23:25:00,422 DEBUG [StoreOpener-1745e7d6d10a6c6039d4c33aadf95a2f-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:35563/user/jenkins/test-data/e9061e95-7d9e-c12a-b652-9e8a1d87f0cf/data/default/testReplayEditsAfterRegionMovedWithMultiCF/1745e7d6d10a6c6039d4c33aadf95a2f/cf1/7d744d5820004695a930452e689f1b2a 2024-12-03T23:25:00,422 INFO [StoreOpener-1745e7d6d10a6c6039d4c33aadf95a2f-1 {}] regionserver.HStore(327): Store=1745e7d6d10a6c6039d4c33aadf95a2f/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T23:25:00,422 INFO [StoreOpener-1745e7d6d10a6c6039d4c33aadf95a2f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 1745e7d6d10a6c6039d4c33aadf95a2f 2024-12-03T23:25:00,423 INFO [StoreOpener-1745e7d6d10a6c6039d4c33aadf95a2f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1745e7d6d10a6c6039d4c33aadf95a2f columnFamilyName cf2 2024-12-03T23:25:00,423 DEBUG [StoreOpener-1745e7d6d10a6c6039d4c33aadf95a2f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T23:25:00,431 DEBUG [StoreOpener-1745e7d6d10a6c6039d4c33aadf95a2f-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:35563/user/jenkins/test-data/e9061e95-7d9e-c12a-b652-9e8a1d87f0cf/data/default/testReplayEditsAfterRegionMovedWithMultiCF/1745e7d6d10a6c6039d4c33aadf95a2f/cf2/fa67e9af7de2471d97906c12a08b2665 2024-12-03T23:25:00,431 INFO [StoreOpener-1745e7d6d10a6c6039d4c33aadf95a2f-1 {}] regionserver.HStore(327): Store=1745e7d6d10a6c6039d4c33aadf95a2f/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T23:25:00,431 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1038): replaying wal for 1745e7d6d10a6c6039d4c33aadf95a2f 2024-12-03T23:25:00,432 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35563/user/jenkins/test-data/e9061e95-7d9e-c12a-b652-9e8a1d87f0cf/data/default/testReplayEditsAfterRegionMovedWithMultiCF/1745e7d6d10a6c6039d4c33aadf95a2f 2024-12-03T23:25:00,435 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35563/user/jenkins/test-data/e9061e95-7d9e-c12a-b652-9e8a1d87f0cf/data/default/testReplayEditsAfterRegionMovedWithMultiCF/1745e7d6d10a6c6039d4c33aadf95a2f 2024-12-03T23:25:00,435 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1048): stopping wal replay for 1745e7d6d10a6c6039d4c33aadf95a2f 2024-12-03T23:25:00,435 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1060): Cleaning up temporary data for 1745e7d6d10a6c6039d4c33aadf95a2f 2024-12-03T23:25:00,436 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-12-03T23:25:00,438 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1093): writing seq id for 1745e7d6d10a6c6039d4c33aadf95a2f 2024-12-03T23:25:00,439 INFO [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1114): Opened 1745e7d6d10a6c6039d4c33aadf95a2f; next sequenceid=18; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=58864211, jitterRate=-0.12285490334033966}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-12-03T23:25:00,439 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1745e7d6d10a6c6039d4c33aadf95a2f 2024-12-03T23:25:00,440 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1006): Region open journal for 1745e7d6d10a6c6039d4c33aadf95a2f: Running coprocessor pre-open hook at 1733268300413Writing region info on filesystem at 1733268300413Initializing all the Stores at 1733268300414 (+1 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733268300414Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733268300414Cleaning up temporary data from old regions at 1733268300435 (+21 ms)Running coprocessor post-open hooks at 1733268300439 (+4 ms)Region opened successfully at 1733268300439 2024-12-03T23:25:00,441 INFO [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1733268298272.1745e7d6d10a6c6039d4c33aadf95a2f., pid=12, masterSystemTime=1733268300408 2024-12-03T23:25:00,444 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1733268298272.1745e7d6d10a6c6039d4c33aadf95a2f. 2024-12-03T23:25:00,444 INFO [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1733268298272.1745e7d6d10a6c6039d4c33aadf95a2f. 2024-12-03T23:25:00,444 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=1745e7d6d10a6c6039d4c33aadf95a2f, regionState=OPEN, openSeqNum=18, regionLocation=5f9d64419733,40969,1733268283535 2024-12-03T23:25:00,447 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=12, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1745e7d6d10a6c6039d4c33aadf95a2f, server=5f9d64419733,40969,1733268283535 because future has completed 2024-12-03T23:25:00,451 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=10 2024-12-03T23:25:00,451 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=10, state=SUCCESS, hasLock=false; OpenRegionProcedure 1745e7d6d10a6c6039d4c33aadf95a2f, server=5f9d64419733,40969,1733268283535 in 193 msec 2024-12-03T23:25:00,453 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=1745e7d6d10a6c6039d4c33aadf95a2f, REOPEN/MOVE in 559 msec 2024-12-03T23:25:00,496 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-03T23:25:00,498 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59512, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-03T23:25:00,500 ERROR [Time-limited test {}] regionserver.HRegionServer(2442): ***** ABORTING region server 5f9d64419733,40969,1733268283535: testing ***** 2024-12-03T23:25:00,500 ERROR [Time-limited test {}] regionserver.HRegionServer(2447): RegionServer abort: loaded coprocessors are: [org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint] 2024-12-03T23:25:00,503 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for java.lang:type=Memory 2024-12-03T23:25:00,504 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=IPC 2024-12-03T23:25:00,506 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Replication 2024-12-03T23:25:00,507 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Server 2024-12-03T23:25:00,517 INFO [Time-limited test {}] regionserver.HRegionServer(2451): Dump of metrics as JSON on abort: { "beans": [ { "name": "java.lang:type=Memory", "modelerType": "sun.management.MemoryImpl", "ObjectPendingFinalizationCount": 0, "HeapMemoryUsage": { "committed": 1048576000, "init": 1048576000, "max": 2306867200, "used": 490413056 }, "NonHeapMemoryUsage": { "committed": 171769856, "init": 7667712, "max": -1, "used": 169129976 }, "Verbose": false, "ObjectName": "java.lang:type=Memory" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=IPC", "modelerType": "RegionServer,sub=IPC", "tag.Context": "regionserver", "tag.Hostname": "5f9d64419733", "queueSize": 0, "numCallsInGeneralQueue": 0, "numCallsInReplicationQueue": 0, "numCallsInBulkLoadQueue": 0, "numCallsInPriorityQueue": 0, "numCallsInMetaPriorityQueue": 0, "numOpenConnections": 3, "numActiveHandler": 0, "numActiveGeneralHandler": 0, "numActivePriorityHandler": 0, "numActiveReplicationHandler": 0, "numGeneralCallsDropped": 0, "numLifoModeSwitches": 0, "numCallsInWriteQueue": 0, "numActiveBulkLoadHandler": 0, "numCallsInReadQueue": 0, "numCallsInScanQueue": 0, "numActiveWriteHandler": 0, "numActiveReadHandler": 0, "numActiveScanHandler": 0, "nettyDirectMemoryUsage": 67108864, "nettyTotalPendingOutboundBytes": 0, "nettyMaxPendingOutboundBytes": 0, "receivedBytes": 2071, "exceptions.RegionMovedException": 0, "authenticationSuccesses": 0, "authorizationFailures": 0, "exceptions.requestTooBig": 0, "UnwritableTime_num_ops": 0, "UnwritableTime_min": 0, "UnwritableTime_max": 0, "UnwritableTime_mean": 0, "UnwritableTime_25th_percentile": 0, "UnwritableTime_median": 0, "UnwritableTime_75th_percentile": 0, "UnwritableTime_90th_percentile": 0, "UnwritableTime_95th_percentile": 0, "UnwritableTime_98th_percentile": 0, "UnwritableTime_99th_percentile": 0, "UnwritableTime_99.9th_percentile": 0, "exceptions.OutOfOrderScannerNextException": 0, "exceptions.rpcThrottling": 0, "exceptions.otherExceptions": 0, "ProcessCallTime_num_ops": 8, "ProcessCallTime_min": 1, "ProcessCallTime_max": 13, "ProcessCallTime_mean": 4, "ProcessCallTime_25th_percentile": 4, "ProcessCallTime_median": 7, "ProcessCallTime_75th_percentile": 10, "ProcessCallTime_90th_percentile": 11, "ProcessCallTime_95th_percentile": 12, "ProcessCallTime_98th_percentile": 12, "ProcessCallTime_99th_percentile": 12, "ProcessCallTime_99.9th_percentile": 12, "ProcessCallTime_TimeRangeCount_0-1": 8, "exceptions.callQueueTooBig": 0, "QueueCallTime_num_ops": 8, "QueueCallTime_min": 0, "QueueCallTime_max": 1, "QueueCallTime_mean": 0, "QueueCallTime_25th_percentile": 0, "QueueCallTime_median": 0, "QueueCallTime_75th_percentile": 0, "QueueCallTime_90th_percentile": 0, "QueueCallTime_95th_percentile": 0, "QueueCallTime_98th_percentile": 0, "QueueCallTime_99th_percentile": 0, "QueueCallTime_99.9th_percentile": 0, "QueueCallTime_TimeRangeCount_0-1": 8, "authenticationFailures": 0, "exceptions.multiResponseTooLarge": 0, "exceptions.callDropped": 0, "TotalCallTime_num_ops": 8, "TotalCallTime_min": 1, "TotalCallTime_max": 13, "TotalCallTime_mean": 4, "TotalCallTime_25th_percentile": 4, "TotalCallTime_median": 7, "TotalCallTime_75th_percentile": 10, "TotalCallTime_90th_percentile": 11, "TotalCallTime_95th_percentile": 12, "TotalCallTime_98th_percentile": 12, "TotalCallTime_99th_percentile": 12, "TotalCallTime_99.9th_percentile": 12, "TotalCallTime_TimeRangeCount_0-1": 8, "exceptions.RegionTooBusyException": 0, "exceptions.FailedSanityCheckException": 0, "ResponseSize_num_ops": 8, "ResponseSize_min": 0, "ResponseSize_max": 175, "ResponseSize_mean": 50, "ResponseSize_25th_percentile": 43, "ResponseSize_median": 87, "ResponseSize_75th_percentile": 131, "ResponseSize_90th_percentile": 157, "ResponseSize_95th_percentile": 166, "ResponseSize_98th_percentile": 171, "ResponseSize_99th_percentile": 173, "ResponseSize_99.9th_percentile": 174, "ResponseSize_SizeRangeCount_0-10": 8, "exceptions.UnknownScannerException": 0, "exceptions": 0, "maxOutboundBytesExceeded": 0, "authenticationFallbacks": 0, "exceptions.quotaExceeded": 0, "exceptions.callTimedOut": 0, "exceptions.NotServingRegionException": 0, "authorizationSuccesses": 0, "exceptions.ScannerResetException": 0, "RequestSize_num_ops": 8, "RequestSize_min": 89, "RequestSize_max": 121, "RequestSize_mean": 103, "RequestSize_25th_percentile": 97, "RequestSize_median": 105, "RequestSize_75th_percentile": 113, "RequestSize_90th_percentile": 117, "RequestSize_95th_percentile": 119, "RequestSize_98th_percentile": 120, "RequestSize_99th_percentile": 120, "RequestSize_99.9th_percentile": 120, "RequestSize_SizeRangeCount_0-10": 8, "sentBytes": 352 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Replication", "modelerType": "RegionServer,sub=Replication", "tag.Context": "regionserver", "tag.Hostname": "5f9d64419733", "source.shippedHFiles": 0, "Source.ageOfLastShippedOp_num_ops": 0, "Source.ageOfLastShippedOp_min": 0, "Source.ageOfLastShippedOp_max": 0, "Source.ageOfLastShippedOp_mean": 0, "Source.ageOfLastShippedOp_25th_percentile": 0, "Source.ageOfLastShippedOp_median": 0, "Source.ageOfLastShippedOp_75th_percentile": 0, "Source.ageOfLastShippedOp_90th_percentile": 0, "Source.ageOfLastShippedOp_95th_percentile": 0, "Source.ageOfLastShippedOp_98th_percentile": 0, "Source.ageOfLastShippedOp_99th_percentile": 0, "Source.ageOfLastShippedOp_99.9th_percentile": 0, "source.uncleanlyClosedLogs": 0, "source.closedLogsWithUnknownFileLength": 0, "source.walReaderEditsBufferUsage": 0, "source.repeatedLogFileBytes": 0, "source.sizeOfHFileRefsQueue": 0, "source.logReadInBytes": 0, "source.completedRecoverQueues": 0, "source.sizeOfLogQueue": 0, "source.restartedLogReading": 0, "source.failedRecoverQueues": 0, "source.ignoredUncleanlyClosedLogContentsInBytes": 0, "Sink.ageOfLastAppliedOp_num_ops": 0, "Sink.ageOfLastAppliedOp_min": 0, "Sink.ageOfLastAppliedOp_max": 0, "Sink.ageOfLastAppliedOp_mean": 0, "Sink.ageOfLastAppliedOp_25th_percentile": 0, "Sink.ageOfLastAppliedOp_median": 0, "Sink.ageOfLastAppliedOp_75th_percentile": 0, "Sink.ageOfLastAppliedOp_90th_percentile": 0, "Sink.ageOfLastAppliedOp_95th_percentile": 0, "Sink.ageOfLastAppliedOp_98th_percentile": 0, "Sink.ageOfLastAppliedOp_99th_percentile": 0, "Sink.ageOfLastAppliedOp_99.9th_percentile": 0, "source.logEditsRead": 0, "source.numInitializing": 0, "source.shippedOps": 0, "sink.appliedHFiles": 0, "source.logEditsFiltered": 0, "source.shippedBytes": 0, "sink.appliedOps": 0, "source.completedLogs": 0, "source.failedBatches": 0, "sink.failedBatches": 0, "source.shippedBatches": 0, "sink.appliedBatches": 0 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Server", "modelerType": "RegionServer,sub=Server", "tag.zookeeperQuorum": "127.0.0.1:51247", "tag.serverName": "5f9d64419733,44215,1733268283642", "tag.clusterId": "c861c93d-86c4-4933-988c-7abbfe26c2c0", "tag.Context": "regionserver", "tag.Hostname": "5f9d64419733", "regionCount": 0, "storeCount": 0, "hlogFileCount": 1, "hlogFileSize": 0, "storeFileCount": 0, "maxStoreFileCount": 0, "memStoreSize": 0, "memStoreHeapSize": 0, "memStoreOffHeapSize": 0, "storeFileSize": 0, "storeFileSizeGrowthRate": 0.0, "maxStoreFileAge": 0, "minStoreFileAge": 0, "avgStoreFileAge": 0, "numReferenceFiles": 0, "regionServerStartTime": 1733268283642, "averageRegionSize": 0, "storeFileIndexSize": 0, "staticIndexSize": 0, "staticBloomSize": 0, "bloomFilterRequestsCount": 0, "bloomFilterNegativeResultsCount": 0, "bloomFilterEligibleRequestsCount": 0, "mutationsWithoutWALCount": 0, "mutationsWithoutWALSize": 0, "percentFilesLocal": 0.0, "percentFilesLocalSecondaryRegions": 0.0, "totalBytesRead": 10034, "localBytesRead": 10034, "shortCircuitBytesRead": 0, "zeroCopyBytesRead": 0, "splitQueueLength": 0, "compactionQueueLength": 0, "smallCompactionQueueLength": 0, "largeCompactionQueueLength": 0, "flushQueueLength": 0, "blockCacheFreeSize": 922070024, "blockCacheCount": 0, "blockCacheDataBlockCount": 0, "blockCacheSize": 676856, "blockCacheCountHitPercent": 0.0, "blockCacheExpressHitPercent": 0.0, "l1CacheSize": 676856, "l1CacheFreeSize": 922070024, "l1CacheCount": 0, "l1CacheEvictionCount": 0, "l1CacheHitCount": 0, "l1CacheMissCount": 0, "l1CacheHitRatio": 0.0, "l1CacheMissRatio": 0.0, "l2CacheSize": 0, "l2CacheFreeSize": 0, "l2CacheCount": 0, "l2CacheEvictionCount": 0, "l2CacheHitCount": 0, "l2CacheMissCount": 0, "l2CacheHitRatio": 0.0, "l2CacheMissRatio": 0.0, "mobFileCacheCount": 0, "mobFileCacheHitPercent": 0.0, "readRequestRatePerSecond": 0.0, "writeRequestRatePerSecond": 0.0, "ByteBuffAllocatorHeapAllocationBytes": 4782, "ByteBuffAllocatorPoolAllocationBytes": 0, "ByteBuffAllocatorHeapAllocationRatio": 0.0, "ByteBuffAllocatorTotalBufferCount": 186, "ByteBuffAllocatorUsedBufferCount": 0, "activeScanners": 0, "totalRequestCount": 0, "totalRowActionRequestCount": 0, "readRequestCount": 0, "cpRequestCount": 0, "filteredReadRequestCount": 0, "writeRequestCount": 0, "rpcGetRequestCount": 0, "rpcFullScanRequestCount": 0, "rpcScanRequestCount": 0, "rpcMultiRequestCount": 0, "rpcMutateRequestCount": 0, "checkMutateFailedCount": 0, "checkMutatePassedCount": 0, "blockCacheHitCount": 0, "blockCacheHitCountPrimary": 0, "blockCacheHitCachingCount": 0, "blockCacheMissCount": 0, "blockCacheMissCountPrimary": 0, "blockCacheMissCachingCount": 0, "blockCacheEvictionCount": 0, "blockCacheEvictionCountPrimary": 0, "blockCacheFailedInsertionCount": 0, "blockCacheDataMissCount": 0, "blockCacheLeafIndexMissCount": 0, "blockCacheBloomChunkMissCount": 0, "blockCacheMetaMissCount": 0, "blockCacheRootIndexMissCount": 0, "blockCacheIntermediateIndexMissCount": 0, "blockCacheFileInfoMissCount": 0, "blockCacheGeneralBloomMetaMissCount": 0, "blockCacheDeleteFamilyBloomMissCount": 0, "blockCacheTrailerMissCount": 0, "blockCacheDataHitCount": 0, "blockCacheLeafIndexHitCount": 0, "blockCacheBloomChunkHitCount": 0, "blockCacheMetaHitCount": 0, "blockCacheRootIndexHitCount": 0, "blockCacheIntermediateIndexHitCount": 0, "blockCacheFileInfoHitCount": 0, "blockCacheGeneralBloomMetaHitCount": 0, "blockCacheDeleteFamilyBloomHitCount": 0, "blockCacheTrailerHitCount": 0, "updatesBlockedTime": 0, "flushedCellsCount": 0, "compactedCellsCount": 0, "majorCompactedCellsCount": 0, "flushedCellsSize": 0, "compactedCellsSize": 0, "majorCompactedCellsSize": 0, "cellsCountCompactedFromMob": 0, "cellsCountCompactedToMob": 0, "cellsSizeCompactedFromMob": 0, "cellsSizeCompactedToMob": 0, "mobFlushCount": 0, "mobFlushedCellsCount": 0, "mobFlushedCellsSize": 0, "mobScanCellsCount": 0, "mobScanCellsSize": 0, "mobFileCacheAccessCount": 0, "mobFileCacheMissCount": 0, "mobFileCacheEvictedCount": 0, "hedgedReads": 0, "hedgedReadWins": 0, "hedgedReadOpsInCurThread": 0, "blockedRequestCount": 0, "CheckAndMutate_num_ops": 0, "CheckAndMutate_min": 0, "CheckAndMutate_max": 0, "CheckAndMutate_mean": 0, "CheckAndMutate_25th_percentile": 0, "CheckAndMutate_median": 0, "CheckAndMutate_75th_percentile": 0, "CheckAndMutate_90th_percentile": 0, "CheckAndMutate_95th_percentile": 0, "CheckAndMutate_98th_percentile": 0, "CheckAndMutate_99th_percentile": 0, "CheckAndMutate_99.9th_percentile": 0, "MajorCompactionTime_num_ops": 0, "MajorCompactionTime_min": 0, "MajorCompactionTime_max": 0, "MajorCompactionTime_mean": 0, "MajorCompactionTime_25th_percentile": 0, "MajorCompactionTime_median": 0, "MajorCompactionTime_75th_percentile": 0, "MajorCompactionTime_90th_percentile": 0, "MajorCompactionTime_95th_percentile": 0, "MajorCompactionTime_98th_percentile": 0, "MajorCompactionTime_99th_percentile": 0, "MajorCompactionTime_99.9th_percentile": 0, "ScanTime_num_ops": 0, "ScanTime_min": 0, "ScanTime_max": 0, "ScanTime_mean": 0, "ScanTime_25th_percentile": 0, "ScanTime_median": 0, "ScanTime_75th_percentile": 0, "ScanTime_90th_percentile": 0, "ScanTime_95th_percentile": 0, "ScanTime_98th_percentile": 0, "ScanTime_99th_percentile": 0, "ScanTime_99.9th_percentile": 0, "CheckAndMutateBlockBytesScanned_num_ops": 0, "CheckAndMutateBlockBytesScanned_min": 0, "CheckAndMutateBlockBytesScanned_max": 0, "CheckAndMutateBlockBytesScanned_mean": 0, "CheckAndMutateBlockBytesScanned_25th_percentile": 0, "CheckAndMutateBlockBytesScanned_median": 0, "CheckAndMutateBlockBytesScanned_75th_percentile": 0, "CheckAndMutateBlockBytesScanned_90th_percentile": 0, "CheckAndMutateBlockBytesScanned_95th_percentile": 0, "CheckAndMutateBlockBytesScanned_98th_percentile": 0, "CheckAndMutateBlockBytesScanned_99th_percentile": 0, "CheckAndMutateBlockBytesScanned_99.9th_percentile": 0, "Put_num_ops": 0, "Put_min": 0, "Put_max": 0, "Put_mean": 0, "Put_25th_percentile": 0, "Put_median": 0, "Put_75th_percentile": 0, "Put_90th_percentile": 0, "Put_95th_percentile": 0, "Put_98th_percentile": 0, "Put_99th_percentile": 0, "Put_99.9th_percentile": 0, "splitRequestCount": 0, "AppendBlockBytesScanned_num_ops": 0, "AppendBlockBytesScanned_min": 0, "AppendBlockBytesScanned_max": 0, "AppendBlockBytesScanned_mean": 0, "AppendBlockBytesScanned_25th_percentile": 0, "AppendBlockBytesScanned_median": 0, "AppendBlockBytesScanned_75th_percentile": 0, "AppendBlockBytesScanned_90th_percentile": 0, "AppendBlockBytesScanned_95th_percentile": 0, "AppendBlockBytesScanned_98th_percentile": 0, "AppendBlockBytesScanned_99th_percentile": 0, "AppendBlockBytesScanned_99.9th_percentile": 0, "PutBatch_num_ops": 0, "PutBatch_min": 0, "PutBatch_max": 0, "PutBatch_mean": 0, "PutBatch_25th_percentile": 0, "PutBatch_median": 0, "PutBatch_75th_percentile": 0, "PutBatch_90th_percentile": 0, "PutBatch_95th_percentile": 0, "PutBatch_98th_percentile": 0, "PutBatch_99th_percentile": 0, "PutBatch_99.9th_percentile": 0, "IncrementBlockBytesScanned_num_ops": 0, "IncrementBlockBytesScanned_min": 0, "IncrementBlockBytesScanned_max": 0, "IncrementBlockBytesScanned_mean": 0, "IncrementBlockBytesScanned_25th_percentile": 0, "IncrementBlockBytesScanned_median": 0, "IncrementBlockBytesScanned_75th_percentile": 0, "IncrementBlockBytesScanned_90th_percentile": 0, "IncrementBlockBytesScanned_95th_percentile": 0, "IncrementBlockBytesScanned_98th_percentile": 0, "IncrementBlockBytesScanned_99th_percentile": 0, "IncrementBlockBytesScanned_99.9th_percentile": 0, "SplitTime_num_ops": 0, "SplitTime_min": 0, "SplitTime_max": 0, "SplitTime_mean": 0, "SplitTime_25th_percentile": 0, "SplitTime_median": 0, "SplitTime_75th_percentile": 0, "SplitTime_90th_percentile": 0, "SplitTime_95th_percentile": 0, "SplitTime_98th_percentile": 0, "SplitTime_99th_percentile": 0, "SplitTime_99.9th_percentile": 0, "GetBlockBytesScanned_num_ops": 0, "GetBlockBytesScanned_min": 0, "GetBlockBytesScanned_max": 0, "GetBlockBytesScanned_mean": 0, "GetBlockBytesScanned_25th_percentile": 0, "GetBlockBytesScanned_median": 0, "GetBlockBytesScanned_75th_percentile": 0, "GetBlockBytesScanned_90th_percentile": 0, "GetBlockBytesScanned_95th_percentile": 0, "GetBlockBytesScanned_98th_percentile": 0, "GetBlockBytesScanned_99th_percentile": 0, "GetBlockBytesScanned_99.9th_percentile": 0, "majorCompactedInputBytes": 0, "slowAppendCount": 0, "flushedOutputBytes": 0, "Replay_num_ops": 0, "Replay_min": 0, "Replay_max": 0, "Replay_mean": 0, "Replay_25th_percentile": 0, "Replay_median": 0, "Replay_75th_percentile": 0, "Replay_90th_percentile": 0, "Replay_95th_percentile": 0, "Replay_98th_percentile": 0, "Replay_99th_percentile": 0, "Replay_99.9th_percentile": 0, "MajorCompactionInputSize_num_ops": 0, "MajorCompactionInputSize_min": 0, "MajorCompactionInputSize_max": 0, "MajorCompactionInputSize_mean": 0, "MajorCompactionInputSize_25th_percentile": 0, "MajorCompactionInputSize_median": 0, "MajorCompactionInputSize_75th_percentile": 0, "MajorCompactionInputSize_90th_percentile": 0, "MajorCompactionInputSize_95th_percentile": 0, "MajorCompactionInputSize_98th_percentile": 0, "MajorCompactionInputSize_99th_percentile": 0, "MajorCompactionInputSize_99.9th_percentile": 0, "pauseInfoThresholdExceeded": 0, "CheckAndDelete_num_ops": 0, "CheckAndDelete_min": 0, "CheckAndDelete_max": 0, "CheckAndDelete_mean": 0, "CheckAndDelete_25th_percentile": 0, "CheckAndDelete_median": 0, "CheckAndDelete_75th_percentile": 0, "CheckAndDelete_90th_percentile": 0, "CheckAndDelete_95th_percentile": 0, "CheckAndDelete_98th_percentile": 0, "CheckAndDelete_99th_percentile": 0, "CheckAndDelete_99.9th_percentile": 0, "CompactionInputSize_num_ops": 0, "CompactionInputSize_min": 0, "CompactionInputSize_max": 0, "CompactionInputSize_mean": 0, "CompactionInputSize_25th_percentile": 0, "CompactionInputSize_median": 0, "CompactionInputSize_75th_percentile": 0, "CompactionInputSize_90th_percentile": 0, "CompactionInputSize_95th_percentile": 0, "CompactionInputSize_98th_percentile": 0, "CompactionInputSize_99th_percentile": 0, "CompactionInputSize_99.9th_percentile": 0, "flushedMemstoreBytes": 0, "majorCompactedOutputBytes": 0, "slowPutCount": 0, "compactedInputBytes": 0, "FlushOutputSize_num_ops": 0, "FlushOutputSize_min": 0, "FlushOutputSize_max": 0, "FlushOutputSize_mean": 0, "FlushOutputSize_25th_percentile": 0, "FlushOutputSize_median": 0, "FlushOutputSize_75th_percentile": 0, "FlushOutputSize_90th_percentile": 0, "FlushOutputSize_95th_percentile": 0, "FlushOutputSize_98th_percentile": 0, "FlushOutputSize_99th_percentile": 0, "FlushOutputSize_99.9th_percentile": 0, "PauseTimeWithGc_num_ops": 0, "PauseTimeWithGc_min": 0, "PauseTimeWithGc_max": 0, "PauseTimeWithGc_mean": 0, "PauseTimeWithGc_25th_percentile": 0, "PauseTimeWithGc_median": 0, "PauseTimeWithGc_75th_percentile": 0, "PauseTimeWithGc_90th_percentile": 0, "PauseTimeWithGc_95th_percentile": 0, "PauseTimeWithGc_98th_percentile": 0, "PauseTimeWithGc_99th_percentile": 0, "PauseTimeWithGc_99.9th_percentile": 0, "compactedOutputBytes": 0, "pauseWarnThresholdExceeded": 0, "ScanBlockBytesScanned_num_ops": 0, "ScanBlockBytesScanned_min": 0, "ScanBlockBytesScanned_max": 0, "ScanBlockBytesScanned_mean": 0, "ScanBlockBytesScanned_25th_percentile": 0, "ScanBlockBytesScanned_median": 0, "ScanBlockBytesScanned_75th_percentile": 0, "ScanBlockBytesScanned_90th_percentile": 0, "ScanBlockBytesScanned_95th_percentile": 0, "ScanBlockBytesScanned_98th_percentile": 0, "ScanBlockBytesScanned_99th_percentile": 0, "ScanBlockBytesScanned_99.9th_percentile": 0, "Increment_num_ops": 0, "Increment_min": 0, "Increment_max": 0, "Increment_mean": 0, "Increment_25th_percentile": 0, "Increment_median": 0, "Increment_75th_percentile": 0, "Increment_90th_percentile": 0, "Increment_95th_percentile": 0, "Increment_98th_percentile": 0, "Increment_99th_percentile": 0, "Increment_99.9th_percentile": 0, "Delete_num_ops": 0, "Delete_min": 0, "Delete_max": 0, "Delete_mean": 0, "Delete_25th_percentile": 0, "Delete_median": 0, "Delete_75th_percentile": 0, "Delete_90th_percentile": 0, "Delete_95th_percentile": 0, "Delete_98th_percentile": 0, "Delete_99th_percentile": 0, "Delete_99.9th_percentile": 0, "DeleteBatch_num_ops": 0, "DeleteBatch_min": 0, "DeleteBatch_max": 0, "DeleteBatch_mean": 0, "DeleteBatch_25th_percentile": 0, "DeleteBatch_median": 0, "DeleteBatch_75th_percentile": 0, "DeleteBatch_90th_percentile": 0, "DeleteBatch_95th_percentile": 0, "DeleteBatch_98th_percentile": 0, "DeleteBatch_99th_percentile": 0, "DeleteBatch_99.9th_percentile": 0, "blockBytesScannedCount": 0, "FlushMemstoreSize_num_ops": 0, "FlushMemstoreSize_min": 0, "FlushMemstoreSize_max": 0, "FlushMemstoreSize_mean": 0, "FlushMemstoreSize_25th_percentile": 0, "FlushMemstoreSize_median": 0, "FlushMemstoreSize_75th_percentile": 0, "FlushMemstoreSize_90th_percentile": 0, "FlushMemstoreSize_95th_percentile": 0, "FlushMemstoreSize_98th_percentile": 0, "FlushMemstoreSize_99th_percentile": 0, "FlushMemstoreSize_99.9th_percentile": 0, "CompactionInputFileCount_num_ops": 0, "CompactionInputFileCount_min": 0, "CompactionInputFileCount_max": 0, "CompactionInputFileCount_mean": 0, "CompactionInputFileCount_25th_percentile": 0, "CompactionInputFileCount_median": 0, "CompactionInputFileCount_75th_percentile": 0, "CompactionInputFileCount_90th_percentile": 0, "CompactionInputFileCount_95th_percentile": 0, "CompactionInputFileCount_98th_percentile": 0, "CompactionInputFileCount_99th_percentile": 0, "CompactionInputFileCount_99.9th_percentile": 0, "CompactionTime_num_ops": 0, "CompactionTime_min": 0, "CompactionTime_max": 0, "CompactionTime_mean": 0, "CompactionTime_25th_percentile": 0, "CompactionTime_median": 0, "CompactionTime_75th_percentile": 0, "CompactionTime_90th_percentile": 0, "CompactionTime_95th_percentile": 0, "CompactionTime_98th_percentile": 0, "CompactionTime_99th_percentile": 0, "CompactionTime_99.9th_percentile": 0, "Get_num_ops": 0, "Get_min": 0, "Get_max": 0, "Get_mean": 0, "Get_25th_percentile": 0, "Get_median": 0, "Get_75th_percentile": 0, "Get_90th_percentile": 0, "Get_95th_percentile": 0, "Get_98th_percentile": 0, "Get_99th_percentile": 0, "Get_99.9th_percentile": 0, "MajorCompactionInputFileCount_num_ops": 0, "MajorCompactionInputFileCount_min": 0, "MajorCompactionInputFileCount_max": 0, "MajorCompactionInputFileCount_mean": 0, "MajorCompactionInputFileCount_25th_percentile": 0, "MajorCompactionInputFileCount_median": 0, "MajorCompactionInputFileCount_75th_percentile": 0, "MajorCompactionInputFileCount_90th_percentile": 0, "MajorCompactionInputFileCount_95th_percentile": 0, "MajorCompactionInputFileCount_98th_percentile": 0, "MajorCompactionInputFileCount_99th_percentile": 0, "MajorCompactionInputFileCount_99.9th_percentile": 0, "scannerLeaseExpiredCount": 0, "CheckAndPut_num_ops": 0, "CheckAndPut_min": 0, "CheckAndPut_max": 0, "CheckAndPut_mean": 0, "CheckAndPut_25th_percentile": 0, "CheckAndPut_median": 0, "CheckAndPut_75th_percentile": 0, "CheckAndPut_90th_percentile": 0, "CheckAndPut_95th_percentile": 0, "CheckAndPut_98th_percentile": 0, "CheckAndPut_99th_percentile": 0, "CheckAndPut_99.9th_percentile": 0, "MajorCompactionOutputSize_num_ops": 0, "MajorCompactionOutputSize_min": 0, "MajorCompactionOutputSize_max": 0, "MajorCompactionOutputSize_mean": 0, "MajorCompactionOutputSize_25th_percentile": 0, "MajorCompactionOutputSize_median": 0, "MajorCompactionOutputSize_75th_percentile": 0, "MajorCompactionOutputSize_90th_percentile": 0, "MajorCompactionOutputSize_95th_percentile": 0, "MajorCompactionOutputSize_98th_percentile": 0, "MajorCompactionOutputSize_99th_percentile": 0, "MajorCompactionOutputSize_99.9th_percentile": 0, "CompactionOutputFileCount_num_ops": 0, "CompactionOutputFileCount_min": 0, "CompactionOutputFileCount_max": 0, "CompactionOutputFileCount_mean": 0, "CompactionOutputFileCount_25th_percentile": 0, "CompactionOutputFileCount_median": 0, "CompactionOutputFileCount_75th_percentile": 0, "CompactionOutputFileCount_90th_percentile": 0, "CompactionOutputFileCount_95th_percentile": 0, "CompactionOutputFileCount_98th_percentile": 0, "CompactionOutputFileCount_99th_percentile": 0, "CompactionOutputFileCount_99.9th_percentile": 0, "slowDeleteCount": 0, "FlushTime_num_ops": 0, "FlushTime_min": 0, "FlushTime_max": 0, "FlushTime_mean": 0, "FlushTime_25th_percentile": 0, "FlushTime_median": 0, "FlushTime_75th_percentile": 0, "FlushTime_90th_percentile": 0, "FlushTime_95th_percentile": 0, "FlushTime_98th_percentile": 0, "FlushTime_99th_percentile": 0, "FlushTime_99.9th_percentile": 0, "splitSuccessCount": 0, "MajorCompactionOutputFileCount_num_ops": 0, "MajorCompactionOutputFileCount_min": 0, "MajorCompactionOutputFileCount_max": 0, "MajorCompactionOutputFileCount_mean": 0, "MajorCompactionOutputFileCount_25th_percentile": 0, "MajorCompactionOutputFileCount_median": 0, "MajorCompactionOutputFileCount_75th_percentile": 0, "MajorCompactionOutputFileCount_90th_percentile": 0, "MajorCompactionOutputFileCount_95th_percentile": 0, "MajorCompactionOutputFileCount_98th_percentile": 0, "MajorCompactionOutputFileCount_99th_percentile": 0, "MajorCompactionOutputFileCount_99.9th_percentile": 0, "slowGetCount": 0, "ScanSize_num_ops": 0, "ScanSize_min": 0, "ScanSize_max": 0, "ScanSize_mean": 0, "ScanSize_25th_percentile": 0, "ScanSize_median": 0, "ScanSize_75th_percentile": 0, "ScanSize_90th_percentile": 0, "ScanSize_95th_percentile": 0, "ScanSize_98th_percentile": 0, "ScanSize_99th_percentile": 0, "ScanSize_99.9th_percentile": 0, "CompactionOutputSize_num_ops": 0, "CompactionOutputSize_min": 0, "CompactionOutputSize_max": 0, "CompactionOutputSize_mean": 0, "CompactionOutputSize_25th_percentile": 0, "CompactionOutputSize_median": 0, "CompactionOutputSize_75th_percentile": 0, "CompactionOutputSize_90th_percentile": 0, "CompactionOutputSize_95th_percentile": 0, "CompactionOutputSize_98th_percentile": 0, "CompactionOutputSize_99th_percentile": 0, "CompactionOutputSize_99.9th_percentile": 0, "PauseTimeWithoutGc_num_ops": 0, "PauseTimeWithoutGc_min": 0, "PauseTimeWithoutGc_max": 0, "PauseTimeWithoutGc_mean": 0, "PauseTimeWithoutGc_25th_percentile": 0, "PauseTimeWithoutGc_median": 0, "PauseTimeWithoutGc_75th_percentile": 0, "PauseTimeWithoutGc_90th_percentile": 0, "PauseTimeWithoutGc_95th_percentile": 0, "PauseTimeWithoutGc_98th_percentile": 0, "PauseTimeWithoutGc_99th_percentile": 0, "PauseTimeWithoutGc_99.9th_percentile": 0, "slowIncrementCount": 0, "Append_num_ops": 0, "Append_min": 0, "Append_max": 0, "Append_mean": 0, "Append_25th_percentile": 0, "Append_median": 0, "Append_75th_percentile": 0, "Append_90th_percentile": 0, "Append_95th_percentile": 0, "Append_98th_percentile": 0, "Append_99th_percentile": 0, "Append_99.9th_percentile": 0, "Bulkload_count": 0, "Bulkload_mean_rate": 0.0, "Bulkload_1min_rate": 0.0, "Bulkload_5min_rate": 0.0, "Bulkload_15min_rate": 0.0, "Bulkload_num_ops": 0, "Bulkload_min": 0, "Bulkload_max": 0, "Bulkload_mean": 0, "Bulkload_25th_percentile": 0, "Bulkload_median": 0, "Bulkload_75th_percentile": 0, "Bulkload_90th_percentile": 0, "Bulkload_95th_percentile": 0, "Bulkload_98th_percentile": 0, "Bulkload_99th_percentile": 0, "Bulkload_99.9th_percentile": 0 } ] } 2024-12-03T23:25:00,522 WARN [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41995 {}] master.MasterRpcServices(700): 5f9d64419733,40969,1733268283535 reported a fatal error: ***** ABORTING region server 5f9d64419733,40969,1733268283535: testing ***** 2024-12-03T23:25:00,524 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '5f9d64419733,40969,1733268283535' ***** 2024-12-03T23:25:00,524 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: testing 2024-12-03T23:25:00,525 INFO [RS:0;5f9d64419733:40969 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-03T23:25:00,525 INFO [RS:0;5f9d64419733:40969 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager abruptly. 2024-12-03T23:25:00,525 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-03T23:25:00,525 INFO [RS:0;5f9d64419733:40969 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager abruptly. 2024-12-03T23:25:00,525 INFO [RS:0;5f9d64419733:40969 {}] regionserver.HRegionServer(3091): Received CLOSE for 1745e7d6d10a6c6039d4c33aadf95a2f 2024-12-03T23:25:00,526 INFO [RS:0;5f9d64419733:40969 {}] regionserver.HRegionServer(956): aborting server 5f9d64419733,40969,1733268283535 2024-12-03T23:25:00,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44215 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Get size: 140 connection: 172.17.0.2:41894 deadline: 1733268360525, exception=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=5f9d64419733 port=40969 startCode=1733268283535. As of locationSeqNum=12. 2024-12-03T23:25:00,526 INFO [RS:0;5f9d64419733:40969 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-03T23:25:00,526 DEBUG [RS_CLOSE_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 1745e7d6d10a6c6039d4c33aadf95a2f, disabling compactions & flushes 2024-12-03T23:25:00,526 INFO [RS:0;5f9d64419733:40969 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;5f9d64419733:40969. 2024-12-03T23:25:00,526 INFO [RS_CLOSE_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1733268298272.1745e7d6d10a6c6039d4c33aadf95a2f. 2024-12-03T23:25:00,526 DEBUG [RS_CLOSE_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733268298272.1745e7d6d10a6c6039d4c33aadf95a2f. 2024-12-03T23:25:00,526 DEBUG [RS:0;5f9d64419733:40969 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-03T23:25:00,526 DEBUG [RS_CLOSE_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733268298272.1745e7d6d10a6c6039d4c33aadf95a2f. after waiting 0 ms 2024-12-03T23:25:00,526 DEBUG [RS_CLOSE_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1733268298272.1745e7d6d10a6c6039d4c33aadf95a2f. 2024-12-03T23:25:00,526 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733268298272.1745e7d6d10a6c6039d4c33aadf95a2f., hostname=5f9d64419733,44215,1733268283642, seqNum=5 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1733268298272.1745e7d6d10a6c6039d4c33aadf95a2f., hostname=5f9d64419733,44215,1733268283642, seqNum=5, error=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=5f9d64419733 port=40969 startCode=1733268283535. As of locationSeqNum=12. 2024-12-03T23:25:00,527 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733268298272.1745e7d6d10a6c6039d4c33aadf95a2f., hostname=5f9d64419733,44215,1733268283642, seqNum=5 is org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=5f9d64419733 port=40969 startCode=1733268283535. As of locationSeqNum=12. 2024-12-03T23:25:00,527 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(84): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733268298272.1745e7d6d10a6c6039d4c33aadf95a2f., hostname=5f9d64419733,44215,1733268283642, seqNum=5 with the new location region=testReplayEditsAfterRegionMovedWithMultiCF,,1733268298272.1745e7d6d10a6c6039d4c33aadf95a2f., hostname=5f9d64419733,40969,1733268283535, seqNum=12 constructed by org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=5f9d64419733 port=40969 startCode=1733268283535. As of locationSeqNum=12. 2024-12-03T23:25:00,527 DEBUG [RS:0;5f9d64419733:40969 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T23:25:00,527 INFO [RS:0;5f9d64419733:40969 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-03T23:25:00,527 INFO [RS:0;5f9d64419733:40969 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-03T23:25:00,527 INFO [RS:0;5f9d64419733:40969 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-03T23:25:00,527 INFO [RS:0;5f9d64419733:40969 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-12-03T23:25:00,528 INFO [RS:0;5f9d64419733:40969 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-12-03T23:25:00,528 DEBUG [RS:0;5f9d64419733:40969 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740, 1745e7d6d10a6c6039d4c33aadf95a2f=testReplayEditsAfterRegionMovedWithMultiCF,,1733268298272.1745e7d6d10a6c6039d4c33aadf95a2f.} 2024-12-03T23:25:00,528 DEBUG [RS_CLOSE_META-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-03T23:25:00,528 INFO [RS_CLOSE_META-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-03T23:25:00,528 DEBUG [RS_CLOSE_META-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-03T23:25:00,528 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server 5f9d64419733,40969,1733268283535 aborting at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processRequest(ServerRpcConnection.java:564) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processOneRpc(ServerRpcConnection.java:364) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyServerRpcConnection.process(NettyServerRpcConnection.java:89) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:56) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:31) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T23:25:00,528 DEBUG [RS_CLOSE_META-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-03T23:25:00,528 DEBUG [RS:0;5f9d64419733:40969 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 1745e7d6d10a6c6039d4c33aadf95a2f 2024-12-03T23:25:00,528 DEBUG [RS_CLOSE_META-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-03T23:25:00,529 ERROR [RS_CLOSE_META-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1960): Memstore data size is 5811 in region hbase:meta,,1.1588230740 2024-12-03T23:25:00,529 INFO [RS_CLOSE_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733268298272.1745e7d6d10a6c6039d4c33aadf95a2f. 2024-12-03T23:25:00,529 DEBUG [RS_CLOSE_META-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-03T23:25:00,529 DEBUG [RS_CLOSE_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 1745e7d6d10a6c6039d4c33aadf95a2f: Waiting for close lock at 1733268300526Running coprocessor pre-close hooks at 1733268300526Disabling compacts and flushes for region at 1733268300526Disabling writes for close at 1733268300526Writing region close event to WAL at 1733268300529 (+3 ms)Running coprocessor post-close hooks at 1733268300529Closed at 1733268300529 2024-12-03T23:25:00,529 INFO [RS_CLOSE_META-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-03T23:25:00,530 DEBUG [RS_CLOSE_META-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733268300528Running coprocessor pre-close hooks at 1733268300528Disabling compacts and flushes for region at 1733268300528Disabling writes for close at 1733268300528Writing region close event to WAL at 1733268300529 (+1 ms)Running coprocessor post-close hooks at 1733268300529Closed at 1733268300529 2024-12-03T23:25:00,530 DEBUG [RS_CLOSE_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733268298272.1745e7d6d10a6c6039d4c33aadf95a2f. 2024-12-03T23:25:00,530 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=5f9d64419733,40969,1733268283535, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=5f9d64419733,40969,1733268283535, seqNum=-1, error=org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server 5f9d64419733,40969,1733268283535 aborting 2024-12-03T23:25:00,530 DEBUG [RS_CLOSE_META-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-12-03T23:25:00,530 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=5f9d64419733,40969,1733268283535, seqNum=-1 is org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server 5f9d64419733,40969,1733268283535 aborting 2024-12-03T23:25:00,530 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=5f9d64419733,40969,1733268283535, seqNum=-1 from cache 2024-12-03T23:25:00,547 INFO [regionserver/5f9d64419733:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-03T23:25:00,641 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T23:25:00,642 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5f9d64419733,40969,1733268283535, seqNum=-1] 2024-12-03T23:25:00,643 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server 5f9d64419733,40969,1733268283535 aborting at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processRequest(ServerRpcConnection.java:564) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processOneRpc(ServerRpcConnection.java:364) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyServerRpcConnection.process(NettyServerRpcConnection.java:89) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:56) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:31) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T23:25:00,644 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=5f9d64419733,40969,1733268283535, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=5f9d64419733,40969,1733268283535, seqNum=-1, error=org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server 5f9d64419733,40969,1733268283535 aborting 2024-12-03T23:25:00,644 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=5f9d64419733,40969,1733268283535, seqNum=-1 is org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server 5f9d64419733,40969,1733268283535 aborting 2024-12-03T23:25:00,644 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=5f9d64419733,40969,1733268283535, seqNum=-1 from cache 2024-12-03T23:25:00,729 INFO [RS:0;5f9d64419733:40969 {}] regionserver.HRegionServer(976): stopping server 5f9d64419733,40969,1733268283535; all regions closed. 2024-12-03T23:25:00,732 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41487 is added to blk_1073741836_1012 (size=2747) 2024-12-03T23:25:00,732 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39829 is added to blk_1073741836_1012 (size=2747) 2024-12-03T23:25:00,732 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45491 is added to blk_1073741836_1012 (size=2747) 2024-12-03T23:25:00,735 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39829 is added to blk_1073741834_1010 (size=1677) 2024-12-03T23:25:00,735 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45491 is added to blk_1073741834_1010 (size=1677) 2024-12-03T23:25:00,735 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41487 is added to blk_1073741834_1010 (size=1677) 2024-12-03T23:25:00,736 DEBUG [RS:0;5f9d64419733:40969 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T23:25:00,736 INFO [RS:0;5f9d64419733:40969 {}] regionserver.LeaseManager(133): Closed leases 2024-12-03T23:25:00,736 INFO [RS:0;5f9d64419733:40969 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-03T23:25:00,736 INFO [RS:0;5f9d64419733:40969 {}] hbase.ChoreService(370): Chore service for: regionserver/5f9d64419733:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-12-03T23:25:00,737 INFO [RS:0;5f9d64419733:40969 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-03T23:25:00,737 INFO [regionserver/5f9d64419733:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-03T23:25:00,737 INFO [RS:0;5f9d64419733:40969 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:40969 2024-12-03T23:25:00,744 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41995-0x1017aa5055b0000, quorum=127.0.0.1:51247, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-03T23:25:00,744 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40969-0x1017aa5055b0001, quorum=127.0.0.1:51247, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/5f9d64419733,40969,1733268283535 2024-12-03T23:25:00,745 INFO [RS:0;5f9d64419733:40969 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-03T23:25:00,746 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [5f9d64419733,40969,1733268283535] 2024-12-03T23:25:00,747 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/5f9d64419733,40969,1733268283535 already deleted, retry=false 2024-12-03T23:25:00,748 INFO [RegionServerTracker-0 {}] master.ServerManager(695): Processing expiration of 5f9d64419733,40969,1733268283535 on 5f9d64419733,41995,1733268282840 2024-12-03T23:25:00,752 DEBUG [RegionServerTracker-0 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure 5f9d64419733,40969,1733268283535, splitWal=true, meta=true 2024-12-03T23:25:00,754 INFO [RegionServerTracker-0 {}] assignment.AssignmentManager(1999): Scheduled ServerCrashProcedure pid=13 for 5f9d64419733,40969,1733268283535 (carryingMeta=true) 5f9d64419733,40969,1733268283535/CRASHED/regionCount=2/lock=java.util.concurrent.locks.ReentrantReadWriteLock@30abf1ad[Write locks = 1, Read locks = 0], oldState=ONLINE. 2024-12-03T23:25:00,755 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(169): Start pid=13, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure 5f9d64419733,40969,1733268283535, splitWal=true, meta=true 2024-12-03T23:25:00,756 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=13, state=RUNNABLE:SERVER_CRASH_SPLIT_META_LOGS, hasLock=true; ServerCrashProcedure 5f9d64419733,40969,1733268283535, splitWal=true, meta=true, isMeta: true 2024-12-03T23:25:00,757 DEBUG [PEWorker-1 {}] master.MasterWalManager(329): Renamed region directory: hdfs://localhost:35563/user/jenkins/test-data/e9061e95-7d9e-c12a-b652-9e8a1d87f0cf/WALs/5f9d64419733,40969,1733268283535-splitting 2024-12-03T23:25:00,758 INFO [PEWorker-1 {}] master.SplitWALManager(105): 5f9d64419733,40969,1733268283535 WAL count=1, meta=true 2024-12-03T23:25:00,761 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure 5f9d64419733%2C40969%2C1733268283535.meta.1733268285441.meta}] 2024-12-03T23:25:00,765 DEBUG [PEWorker-2 {}] master.SplitWALManager(158): Acquired split WAL worker=5f9d64419733,44215,1733268283642 2024-12-03T23:25:00,767 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=15, ppid=14, state=RUNNABLE, hasLock=false; SplitWALRemoteProcedure 5f9d64419733%2C40969%2C1733268283535.meta.1733268285441.meta, worker=5f9d64419733,44215,1733268283642}] 2024-12-03T23:25:00,847 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40969-0x1017aa5055b0001, quorum=127.0.0.1:51247, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-03T23:25:00,847 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40969-0x1017aa5055b0001, quorum=127.0.0.1:51247, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-03T23:25:00,848 INFO [RS:0;5f9d64419733:40969 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-03T23:25:00,848 INFO [RS:0;5f9d64419733:40969 {}] regionserver.HRegionServer(1031): Exiting; stopping=5f9d64419733,40969,1733268283535; zookeeper connection closed. 2024-12-03T23:25:00,848 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@76f32c63 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@76f32c63 2024-12-03T23:25:00,850 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T23:25:00,852 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5f9d64419733,40969,1733268283535, seqNum=-1] 2024-12-03T23:25:00,853 WARN [RPCClient-NioEventLoopGroup-6-3 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 5f9d64419733:40969 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: 5f9d64419733/172.17.0.2:40969 Caused by: java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hbase.thirdparty.io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:336) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:339) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T23:25:00,854 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=5f9d64419733,40969,1733268283535, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=5f9d64419733,40969,1733268283535, seqNum=-1, error=java.net.ConnectException: Call to address=5f9d64419733:40969 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: 5f9d64419733/172.17.0.2:40969 2024-12-03T23:25:00,854 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=5f9d64419733,40969,1733268283535, seqNum=-1 is java.net.ConnectException: Connection refused 2024-12-03T23:25:00,854 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=5f9d64419733,40969,1733268283535, seqNum=-1 from cache 2024-12-03T23:25:00,854 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] ipc.FailedServers(52): Added failed server with address 5f9d64419733:40969 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: 5f9d64419733/172.17.0.2:40969 2024-12-03T23:25:00,930 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44215 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SplitWALCallable, pid=15 2024-12-03T23:25:00,948 INFO [RS_LOG_REPLAY_OPS-regionserver/5f9d64419733:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(299): Splitting hdfs://localhost:35563/user/jenkins/test-data/e9061e95-7d9e-c12a-b652-9e8a1d87f0cf/WALs/5f9d64419733,40969,1733268283535-splitting/5f9d64419733%2C40969%2C1733268283535.meta.1733268285441.meta, size=2.7 K (2747bytes) 2024-12-03T23:25:00,948 INFO [RS_LOG_REPLAY_OPS-regionserver/5f9d64419733:0-0 {event_type=RS_LOG_REPLAY, pid=15}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:35563/user/jenkins/test-data/e9061e95-7d9e-c12a-b652-9e8a1d87f0cf/WALs/5f9d64419733,40969,1733268283535-splitting/5f9d64419733%2C40969%2C1733268283535.meta.1733268285441.meta 2024-12-03T23:25:00,949 INFO [RS_LOG_REPLAY_OPS-regionserver/5f9d64419733:0-0 {event_type=RS_LOG_REPLAY, pid=15}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:35563/user/jenkins/test-data/e9061e95-7d9e-c12a-b652-9e8a1d87f0cf/WALs/5f9d64419733,40969,1733268283535-splitting/5f9d64419733%2C40969%2C1733268283535.meta.1733268285441.meta after 1ms 2024-12-03T23:25:00,951 DEBUG [RS_LOG_REPLAY_OPS-regionserver/5f9d64419733:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:35563/user/jenkins/test-data/e9061e95-7d9e-c12a-b652-9e8a1d87f0cf/WALs/5f9d64419733,40969,1733268283535-splitting/5f9d64419733%2C40969%2C1733268283535.meta.1733268285441.meta: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-03T23:25:00,952 INFO [RS_LOG_REPLAY_OPS-regionserver/5f9d64419733:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(310): Open hdfs://localhost:35563/user/jenkins/test-data/e9061e95-7d9e-c12a-b652-9e8a1d87f0cf/WALs/5f9d64419733,40969,1733268283535-splitting/5f9d64419733%2C40969%2C1733268283535.meta.1733268285441.meta took 3ms 2024-12-03T23:25:00,959 DEBUG [RS_LOG_REPLAY_OPS-regionserver/5f9d64419733:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(352): Last flushed sequenceid for 1588230740: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "info" sequence_id: 5 } store_sequence_id { family_name: "ns" sequence_id: 3 } store_sequence_id { family_name: "rep_barrier" sequence_id: 18446744073709551615 } store_sequence_id { family_name: "table" sequence_id: 6 } 2024-12-03T23:25:00,960 DEBUG [RS_LOG_REPLAY_OPS-regionserver/5f9d64419733:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:35563/user/jenkins/test-data/e9061e95-7d9e-c12a-b652-9e8a1d87f0cf/WALs/5f9d64419733,40969,1733268283535-splitting/5f9d64419733%2C40969%2C1733268283535.meta.1733268285441.meta so closing down 2024-12-03T23:25:00,960 DEBUG [RS_LOG_REPLAY_OPS-regionserver/5f9d64419733:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-03T23:25:00,961 INFO [RS_LOG_REPLAY_OPS-regionserver/5f9d64419733:0-0-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-5f9d64419733%2C40969%2C1733268283535.meta.1733268285441.meta.temp 2024-12-03T23:25:00,962 INFO [RS_LOG_REPLAY_OPS-regionserver/5f9d64419733:0-0-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:35563/user/jenkins/test-data/e9061e95-7d9e-c12a-b652-9e8a1d87f0cf/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-5f9d64419733%2C40969%2C1733268283535.meta.1733268285441.meta.temp 2024-12-03T23:25:00,963 INFO [RS_LOG_REPLAY_OPS-regionserver/5f9d64419733:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.OutputSink(145): 3 split writer threads finished 2024-12-03T23:25:00,970 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41487 is added to blk_1073741892_1070 (size=2527) 2024-12-03T23:25:00,970 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45491 is added to blk_1073741892_1070 (size=2527) 2024-12-03T23:25:00,972 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39829 is added to blk_1073741892_1070 (size=2527) 2024-12-03T23:25:00,972 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:35563/user/jenkins/test-data/e9061e95-7d9e-c12a-b652-9e8a1d87f0cf/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-5f9d64419733%2C40969%2C1733268283535.meta.1733268285441.meta.temp (wrote 15 edits, skipped 0 edits in 0 ms) 2024-12-03T23:25:00,974 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:35563/user/jenkins/test-data/e9061e95-7d9e-c12a-b652-9e8a1d87f0cf/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-5f9d64419733%2C40969%2C1733268283535.meta.1733268285441.meta.temp to hdfs://localhost:35563/user/jenkins/test-data/e9061e95-7d9e-c12a-b652-9e8a1d87f0cf/data/hbase/meta/1588230740/recovered.edits/0000000000000000018 2024-12-03T23:25:00,974 INFO [RS_LOG_REPLAY_OPS-regionserver/5f9d64419733:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(425): Processed 16 edits across 1 Regions in 22 ms; skipped=1; WAL=hdfs://localhost:35563/user/jenkins/test-data/e9061e95-7d9e-c12a-b652-9e8a1d87f0cf/WALs/5f9d64419733,40969,1733268283535-splitting/5f9d64419733%2C40969%2C1733268283535.meta.1733268285441.meta, size=2.7 K, length=2747, corrupted=false, cancelled=false 2024-12-03T23:25:00,974 DEBUG [RS_LOG_REPLAY_OPS-regionserver/5f9d64419733:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(428): Completed split of hdfs://localhost:35563/user/jenkins/test-data/e9061e95-7d9e-c12a-b652-9e8a1d87f0cf/WALs/5f9d64419733,40969,1733268283535-splitting/5f9d64419733%2C40969%2C1733268283535.meta.1733268285441.meta, journal: Splitting hdfs://localhost:35563/user/jenkins/test-data/e9061e95-7d9e-c12a-b652-9e8a1d87f0cf/WALs/5f9d64419733,40969,1733268283535-splitting/5f9d64419733%2C40969%2C1733268283535.meta.1733268285441.meta, size=2.7 K (2747bytes) at 1733268300948Finishing writing output for hdfs://localhost:35563/user/jenkins/test-data/e9061e95-7d9e-c12a-b652-9e8a1d87f0cf/WALs/5f9d64419733,40969,1733268283535-splitting/5f9d64419733%2C40969%2C1733268283535.meta.1733268285441.meta so closing down at 1733268300960 (+12 ms)Creating recovered edits writer path=hdfs://localhost:35563/user/jenkins/test-data/e9061e95-7d9e-c12a-b652-9e8a1d87f0cf/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-5f9d64419733%2C40969%2C1733268283535.meta.1733268285441.meta.temp at 1733268300962 (+2 ms)3 split writer threads finished at 1733268300963 (+1 ms)Closed recovered edits writer path=hdfs://localhost:35563/user/jenkins/test-data/e9061e95-7d9e-c12a-b652-9e8a1d87f0cf/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-5f9d64419733%2C40969%2C1733268283535.meta.1733268285441.meta.temp (wrote 15 edits, skipped 0 edits in 0 ms) at 1733268300972 (+9 ms)Rename recovered edits hdfs://localhost:35563/user/jenkins/test-data/e9061e95-7d9e-c12a-b652-9e8a1d87f0cf/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-5f9d64419733%2C40969%2C1733268283535.meta.1733268285441.meta.temp to hdfs://localhost:35563/user/jenkins/test-data/e9061e95-7d9e-c12a-b652-9e8a1d87f0cf/data/hbase/meta/1588230740/recovered.edits/0000000000000000018 at 1733268300974 (+2 ms)Processed 16 edits across 1 Regions in 22 ms; skipped=1; WAL=hdfs://localhost:35563/user/jenkins/test-data/e9061e95-7d9e-c12a-b652-9e8a1d87f0cf/WALs/5f9d64419733,40969,1733268283535-splitting/5f9d64419733%2C40969%2C1733268283535.meta.1733268285441.meta, size=2.7 K, length=2747, corrupted=false, cancelled=false at 1733268300974 2024-12-03T23:25:00,974 DEBUG [RS_LOG_REPLAY_OPS-regionserver/5f9d64419733:0-0 {event_type=RS_LOG_REPLAY, pid=15}] regionserver.SplitLogWorker(218): Done splitting WAL hdfs://localhost:35563/user/jenkins/test-data/e9061e95-7d9e-c12a-b652-9e8a1d87f0cf/WALs/5f9d64419733,40969,1733268283535-splitting/5f9d64419733%2C40969%2C1733268283535.meta.1733268285441.meta 2024-12-03T23:25:00,976 DEBUG [RS_LOG_REPLAY_OPS-regionserver/5f9d64419733:0-0 {event_type=RS_LOG_REPLAY, pid=15}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=15 2024-12-03T23:25:00,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41995 {}] master.HMaster(4169): Remote procedure done, pid=15 2024-12-03T23:25:00,983 INFO [PEWorker-4 {}] wal.WALSplitUtil(143): Moved hdfs://localhost:35563/user/jenkins/test-data/e9061e95-7d9e-c12a-b652-9e8a1d87f0cf/WALs/5f9d64419733,40969,1733268283535-splitting/5f9d64419733%2C40969%2C1733268283535.meta.1733268285441.meta to hdfs://localhost:35563/user/jenkins/test-data/e9061e95-7d9e-c12a-b652-9e8a1d87f0cf/oldWALs 2024-12-03T23:25:00,987 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=15, resume processing ppid=14 2024-12-03T23:25:00,987 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, ppid=14, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 5f9d64419733%2C40969%2C1733268283535.meta.1733268285441.meta, worker=5f9d64419733,44215,1733268283642 in 216 msec 2024-12-03T23:25:00,988 DEBUG [PEWorker-5 {}] master.SplitWALManager(172): Release split WAL worker=5f9d64419733,44215,1733268283642 2024-12-03T23:25:00,991 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-12-03T23:25:00,991 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; SplitWALProcedure 5f9d64419733%2C40969%2C1733268283535.meta.1733268285441.meta, worker=5f9d64419733,44215,1733268283642 in 229 msec 2024-12-03T23:25:00,993 INFO [PEWorker-1 {}] master.SplitLogManager(171): hdfs://localhost:35563/user/jenkins/test-data/e9061e95-7d9e-c12a-b652-9e8a1d87f0cf/WALs/5f9d64419733,40969,1733268283535-splitting dir is empty, no logs to split. 2024-12-03T23:25:00,993 INFO [PEWorker-1 {}] master.SplitWALManager(105): 5f9d64419733,40969,1733268283535 WAL count=0, meta=true 2024-12-03T23:25:00,993 DEBUG [PEWorker-1 {}] procedure.ServerCrashProcedure(329): Check if 5f9d64419733,40969,1733268283535 WAL splitting is done? wals=0, meta=true 2024-12-03T23:25:00,995 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-12-03T23:25:00,996 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-03T23:25:00,997 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-12-03T23:25:01,147 DEBUG [5f9d64419733:41995 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-12-03T23:25:01,147 DEBUG [5f9d64419733:41995 {}] balancer.BalancerClusterState(204): Hosts are {5f9d64419733=0} racks are {/default-rack=0} 2024-12-03T23:25:01,148 DEBUG [5f9d64419733:41995 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-12-03T23:25:01,148 DEBUG [5f9d64419733:41995 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-12-03T23:25:01,148 DEBUG [5f9d64419733:41995 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-03T23:25:01,148 DEBUG [5f9d64419733:41995 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-03T23:25:01,148 INFO [5f9d64419733:41995 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-03T23:25:01,148 INFO [5f9d64419733:41995 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-03T23:25:01,148 DEBUG [5f9d64419733:41995 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-12-03T23:25:01,148 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=5f9d64419733,44215,1733268283642 2024-12-03T23:25:01,150 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 5f9d64419733,44215,1733268283642, state=OPENING 2024-12-03T23:25:01,154 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41995-0x1017aa5055b0000, quorum=127.0.0.1:51247, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-03T23:25:01,154 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44411-0x1017aa5055b0003, quorum=127.0.0.1:51247, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-03T23:25:01,154 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44215-0x1017aa5055b0002, quorum=127.0.0.1:51247, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-03T23:25:01,154 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-03T23:25:01,154 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T23:25:01,154 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T23:25:01,154 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T23:25:01,154 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=17, ppid=16, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=5f9d64419733,44215,1733268283642}] 2024-12-03T23:25:01,161 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T23:25:01,162 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5f9d64419733,44215,1733268283642, seqNum=-1] 2024-12-03T23:25:01,163 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44215 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Get size: 119 connection: 172.17.0.2:41894 deadline: 1733268361163, exception=org.apache.hadoop.hbase.NotServingRegionException: hbase:meta,,1 is not online on 5f9d64419733,44215,1733268283642 2024-12-03T23:25:01,164 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=5f9d64419733,44215,1733268283642, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=5f9d64419733,44215,1733268283642, seqNum=-1, error=org.apache.hadoop.hbase.NotServingRegionException: org.apache.hadoop.hbase.NotServingRegionException: hbase:meta,,1 is not online on 5f9d64419733,44215,1733268283642 at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:3186) at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:3164) at org.apache.hadoop.hbase.regionserver.RSRpcServices.getRegion(RSRpcServices.java:1413) at org.apache.hadoop.hbase.regionserver.RSRpcServices.get(RSRpcServices.java:2474) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43504) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T23:25:01,164 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=5f9d64419733,44215,1733268283642, seqNum=-1 is org.apache.hadoop.hbase.NotServingRegionException: org.apache.hadoop.hbase.NotServingRegionException: hbase:meta,,1 is not online on 5f9d64419733,44215,1733268283642 at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:3186) at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:3164) at org.apache.hadoop.hbase.regionserver.RSRpcServices.getRegion(RSRpcServices.java:1413) at org.apache.hadoop.hbase.regionserver.RSRpcServices.get(RSRpcServices.java:2474) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43504) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T23:25:01,164 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=5f9d64419733,44215,1733268283642, seqNum=-1 from cache 2024-12-03T23:25:01,311 INFO [RS_OPEN_META-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_META, pid=17}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-12-03T23:25:01,312 INFO [RS_OPEN_META-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_META, pid=17}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-03T23:25:01,312 INFO [RS_OPEN_META-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_META, pid=17}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-12-03T23:25:01,314 INFO [RS_OPEN_META-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_META, pid=17}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=5f9d64419733%2C44215%2C1733268283642.meta, suffix=.meta, logDir=hdfs://localhost:35563/user/jenkins/test-data/e9061e95-7d9e-c12a-b652-9e8a1d87f0cf/WALs/5f9d64419733,44215,1733268283642, archiveDir=hdfs://localhost:35563/user/jenkins/test-data/e9061e95-7d9e-c12a-b652-9e8a1d87f0cf/oldWALs, maxLogs=32 2024-12-03T23:25:01,328 DEBUG [RS_OPEN_META-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_META, pid=17}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/e9061e95-7d9e-c12a-b652-9e8a1d87f0cf/WALs/5f9d64419733,44215,1733268283642/5f9d64419733%2C44215%2C1733268283642.meta.1733268301314.meta, exclude list is [], retry=0 2024-12-03T23:25:01,331 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41487,DS-459d168f-61f7-403b-9139-91eca369c54e,DISK] 2024-12-03T23:25:01,331 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39829,DS-e6270720-ef81-458e-b5d6-959d16c4c378,DISK] 2024-12-03T23:25:01,331 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45491,DS-d55e5535-c0ce-4dc2-a45c-8e70eb010e56,DISK] 2024-12-03T23:25:01,334 INFO [RS_OPEN_META-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_META, pid=17}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/e9061e95-7d9e-c12a-b652-9e8a1d87f0cf/WALs/5f9d64419733,44215,1733268283642/5f9d64419733%2C44215%2C1733268283642.meta.1733268301314.meta 2024-12-03T23:25:01,335 DEBUG [RS_OPEN_META-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_META, pid=17}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36361:36361),(127.0.0.1/127.0.0.1:38065:38065),(127.0.0.1/127.0.0.1:34451:34451)] 2024-12-03T23:25:01,335 DEBUG [RS_OPEN_META-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-12-03T23:25:01,335 DEBUG [RS_OPEN_META-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_META, pid=17}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-12-03T23:25:01,335 DEBUG [RS_OPEN_META-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-12-03T23:25:01,336 INFO [RS_OPEN_META-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-12-03T23:25:01,336 DEBUG [RS_OPEN_META-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-12-03T23:25:01,336 DEBUG [RS_OPEN_META-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T23:25:01,336 DEBUG [RS_OPEN_META-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-12-03T23:25:01,336 DEBUG [RS_OPEN_META-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-12-03T23:25:01,337 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-03T23:25:01,338 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-03T23:25:01,339 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T23:25:01,339 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-03T23:25:01,339 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-03T23:25:01,340 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-03T23:25:01,340 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T23:25:01,341 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-03T23:25:01,341 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-03T23:25:01,342 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-03T23:25:01,342 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T23:25:01,342 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-03T23:25:01,342 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-03T23:25:01,343 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-03T23:25:01,343 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T23:25:01,344 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-03T23:25:01,344 DEBUG [RS_OPEN_META-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-03T23:25:01,345 DEBUG [RS_OPEN_META-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35563/user/jenkins/test-data/e9061e95-7d9e-c12a-b652-9e8a1d87f0cf/data/hbase/meta/1588230740 2024-12-03T23:25:01,346 DEBUG [RS_OPEN_META-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:35563/user/jenkins/test-data/e9061e95-7d9e-c12a-b652-9e8a1d87f0cf/data/hbase/meta/1588230740 2024-12-03T23:25:01,347 INFO [RS_OPEN_META-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:35563/user/jenkins/test-data/e9061e95-7d9e-c12a-b652-9e8a1d87f0cf/data/hbase/meta/1588230740/recovered.edits/0000000000000000018 2024-12-03T23:25:01,349 DEBUG [RS_OPEN_META-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_META, pid=17}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:35563/user/jenkins/test-data/e9061e95-7d9e-c12a-b652-9e8a1d87f0cf/data/hbase/meta/1588230740/recovered.edits/0000000000000000018: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-03T23:25:01,352 DEBUG [RS_OPEN_META-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(5793): Applied 40, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=18, path=hdfs://localhost:35563/user/jenkins/test-data/e9061e95-7d9e-c12a-b652-9e8a1d87f0cf/data/hbase/meta/1588230740/recovered.edits/0000000000000000018 2024-12-03T23:25:01,352 INFO [RS_OPEN_META-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=5.67 KB heapSize=9.66 KB 2024-12-03T23:25:01,373 DEBUG [RS_OPEN_META-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_META, pid=17}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35563/user/jenkins/test-data/e9061e95-7d9e-c12a-b652-9e8a1d87f0cf/data/hbase/meta/1588230740/.tmp/info/89b1249ca5804c718d32d4851303ef7c is 205, key is testReplayEditsAfterRegionMovedWithMultiCF,,1733268298272.1745e7d6d10a6c6039d4c33aadf95a2f./info:regioninfo/1733268300444/Put/seqid=0 2024-12-03T23:25:01,382 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45491 is added to blk_1073741894_1072 (size=11177) 2024-12-03T23:25:01,383 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39829 is added to blk_1073741894_1072 (size=11177) 2024-12-03T23:25:01,383 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41487 is added to blk_1073741894_1072 (size=11177) 2024-12-03T23:25:01,387 INFO [RS_OPEN_META-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=5.46 KB at sequenceid=18 (bloomFilter=true), to=hdfs://localhost:35563/user/jenkins/test-data/e9061e95-7d9e-c12a-b652-9e8a1d87f0cf/data/hbase/meta/1588230740/.tmp/info/89b1249ca5804c718d32d4851303ef7c 2024-12-03T23:25:01,409 DEBUG [RS_OPEN_META-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_META, pid=17}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35563/user/jenkins/test-data/e9061e95-7d9e-c12a-b652-9e8a1d87f0cf/data/hbase/meta/1588230740/.tmp/ns/20352d5f7fb54520a2597243b9efb569 is 43, key is default/ns:d/1733268285643/Put/seqid=0 2024-12-03T23:25:01,415 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41487 is added to blk_1073741895_1073 (size=5153) 2024-12-03T23:25:01,416 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45491 is added to blk_1073741895_1073 (size=5153) 2024-12-03T23:25:01,416 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39829 is added to blk_1073741895_1073 (size=5153) 2024-12-03T23:25:01,417 INFO [RS_OPEN_META-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=18 (bloomFilter=true), to=hdfs://localhost:35563/user/jenkins/test-data/e9061e95-7d9e-c12a-b652-9e8a1d87f0cf/data/hbase/meta/1588230740/.tmp/ns/20352d5f7fb54520a2597243b9efb569 2024-12-03T23:25:01,438 DEBUG [RS_OPEN_META-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_META, pid=17}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35563/user/jenkins/test-data/e9061e95-7d9e-c12a-b652-9e8a1d87f0cf/data/hbase/meta/1588230740/.tmp/table/0ad0179ec6a0435d94486bd856c6bb8d is 78, key is testReplayEditsAfterRegionMovedWithMultiCF/table:state/1733268298687/Put/seqid=0 2024-12-03T23:25:01,444 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39829 is added to blk_1073741896_1074 (size=5431) 2024-12-03T23:25:01,444 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41487 is added to blk_1073741896_1074 (size=5431) 2024-12-03T23:25:01,444 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45491 is added to blk_1073741896_1074 (size=5431) 2024-12-03T23:25:01,445 INFO [RS_OPEN_META-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=148 B at sequenceid=18 (bloomFilter=true), to=hdfs://localhost:35563/user/jenkins/test-data/e9061e95-7d9e-c12a-b652-9e8a1d87f0cf/data/hbase/meta/1588230740/.tmp/table/0ad0179ec6a0435d94486bd856c6bb8d 2024-12-03T23:25:01,451 DEBUG [RS_OPEN_META-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35563/user/jenkins/test-data/e9061e95-7d9e-c12a-b652-9e8a1d87f0cf/data/hbase/meta/1588230740/.tmp/info/89b1249ca5804c718d32d4851303ef7c as hdfs://localhost:35563/user/jenkins/test-data/e9061e95-7d9e-c12a-b652-9e8a1d87f0cf/data/hbase/meta/1588230740/info/89b1249ca5804c718d32d4851303ef7c 2024-12-03T23:25:01,457 INFO [RS_OPEN_META-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35563/user/jenkins/test-data/e9061e95-7d9e-c12a-b652-9e8a1d87f0cf/data/hbase/meta/1588230740/info/89b1249ca5804c718d32d4851303ef7c, entries=36, sequenceid=18, filesize=10.9 K 2024-12-03T23:25:01,459 DEBUG [RS_OPEN_META-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35563/user/jenkins/test-data/e9061e95-7d9e-c12a-b652-9e8a1d87f0cf/data/hbase/meta/1588230740/.tmp/ns/20352d5f7fb54520a2597243b9efb569 as hdfs://localhost:35563/user/jenkins/test-data/e9061e95-7d9e-c12a-b652-9e8a1d87f0cf/data/hbase/meta/1588230740/ns/20352d5f7fb54520a2597243b9efb569 2024-12-03T23:25:01,465 INFO [RS_OPEN_META-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35563/user/jenkins/test-data/e9061e95-7d9e-c12a-b652-9e8a1d87f0cf/data/hbase/meta/1588230740/ns/20352d5f7fb54520a2597243b9efb569, entries=2, sequenceid=18, filesize=5.0 K 2024-12-03T23:25:01,466 DEBUG [RS_OPEN_META-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35563/user/jenkins/test-data/e9061e95-7d9e-c12a-b652-9e8a1d87f0cf/data/hbase/meta/1588230740/.tmp/table/0ad0179ec6a0435d94486bd856c6bb8d as hdfs://localhost:35563/user/jenkins/test-data/e9061e95-7d9e-c12a-b652-9e8a1d87f0cf/data/hbase/meta/1588230740/table/0ad0179ec6a0435d94486bd856c6bb8d 2024-12-03T23:25:01,472 INFO [RS_OPEN_META-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35563/user/jenkins/test-data/e9061e95-7d9e-c12a-b652-9e8a1d87f0cf/data/hbase/meta/1588230740/table/0ad0179ec6a0435d94486bd856c6bb8d, entries=2, sequenceid=18, filesize=5.3 K 2024-12-03T23:25:01,472 INFO [RS_OPEN_META-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(3140): Finished flush of dataSize ~5.67 KB/5811, heapSize ~9.37 KB/9592, currentSize=0 B/0 for 1588230740 in 120ms, sequenceid=18, compaction requested=false; wal=null 2024-12-03T23:25:01,473 DEBUG [RS_OPEN_META-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:35563/user/jenkins/test-data/e9061e95-7d9e-c12a-b652-9e8a1d87f0cf/data/hbase/meta/1588230740/recovered.edits/0000000000000000018 2024-12-03T23:25:01,474 DEBUG [RS_OPEN_META-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-03T23:25:01,474 DEBUG [RS_OPEN_META-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-03T23:25:01,475 DEBUG [RS_OPEN_META-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-03T23:25:01,477 DEBUG [RS_OPEN_META-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-03T23:25:01,479 DEBUG [RS_OPEN_META-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_META, pid=17}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35563/user/jenkins/test-data/e9061e95-7d9e-c12a-b652-9e8a1d87f0cf/data/hbase/meta/1588230740/recovered.edits/18.seqid, newMaxSeqId=18, maxSeqId=1 2024-12-03T23:25:01,480 INFO [RS_OPEN_META-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=19; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69068597, jitterRate=0.02920229732990265}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-03T23:25:01,480 DEBUG [RS_OPEN_META-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-12-03T23:25:01,481 DEBUG [RS_OPEN_META-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1733268301336Writing region info on filesystem at 1733268301336Initializing all the Stores at 1733268301337 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733268301337Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733268301337Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733268301337Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733268301337Obtaining lock to block concurrent updates at 1733268301352 (+15 ms)Preparing flush snapshotting stores in 1588230740 at 1733268301352Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=5811, getHeapSize=9832, getOffHeapSize=0, getCellsCount=40 at 1733268301352Flushing stores of hbase:meta,,1.1588230740 at 1733268301352Flushing 1588230740/info: creating writer at 1733268301352Flushing 1588230740/info: appending metadata at 1733268301373 (+21 ms)Flushing 1588230740/info: closing flushed file at 1733268301373Flushing 1588230740/ns: creating writer at 1733268301393 (+20 ms)Flushing 1588230740/ns: appending metadata at 1733268301408 (+15 ms)Flushing 1588230740/ns: closing flushed file at 1733268301408Flushing 1588230740/table: creating writer at 1733268301423 (+15 ms)Flushing 1588230740/table: appending metadata at 1733268301437 (+14 ms)Flushing 1588230740/table: closing flushed file at 1733268301437Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7197984c: reopening flushed file at 1733268301450 (+13 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@74a54bd4: reopening flushed file at 1733268301458 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@203eb636: reopening flushed file at 1733268301465 (+7 ms)Finished flush of dataSize ~5.67 KB/5811, heapSize ~9.37 KB/9592, currentSize=0 B/0 for 1588230740 in 120ms, sequenceid=18, compaction requested=false; wal=null at 1733268301472 (+7 ms)Cleaning up temporary data from old regions at 1733268301474 (+2 ms)Running coprocessor post-open hooks at 1733268301480 (+6 ms)Region opened successfully at 1733268301481 (+1 ms) 2024-12-03T23:25:01,482 INFO [RS_OPEN_META-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=17, masterSystemTime=1733268301307 2024-12-03T23:25:01,484 DEBUG [RS_OPEN_META-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-12-03T23:25:01,484 INFO [RS_OPEN_META-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_META, pid=17}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-12-03T23:25:01,485 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=19, regionLocation=5f9d64419733,44215,1733268283642 2024-12-03T23:25:01,486 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 5f9d64419733,44215,1733268283642, state=OPEN 2024-12-03T23:25:01,488 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44215-0x1017aa5055b0002, quorum=127.0.0.1:51247, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-03T23:25:01,488 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44411-0x1017aa5055b0003, quorum=127.0.0.1:51247, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-03T23:25:01,488 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41995-0x1017aa5055b0000, quorum=127.0.0.1:51247, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-03T23:25:01,488 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T23:25:01,488 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=17, ppid=16, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=5f9d64419733,44215,1733268283642 2024-12-03T23:25:01,488 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T23:25:01,488 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T23:25:01,491 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=17, resume processing ppid=16 2024-12-03T23:25:01,491 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, ppid=16, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=5f9d64419733,44215,1733268283642 in 334 msec 2024-12-03T23:25:01,493 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=13 2024-12-03T23:25:01,493 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=13, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 496 msec 2024-12-03T23:25:01,494 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(207): 5f9d64419733,40969,1733268283535 had 2 regions 2024-12-03T23:25:01,495 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=13, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure 5f9d64419733,40969,1733268283535, splitWal=true, meta=true, isMeta: false 2024-12-03T23:25:01,497 INFO [PEWorker-2 {}] master.SplitWALManager(105): 5f9d64419733,40969,1733268283535 WAL count=1, meta=false 2024-12-03T23:25:01,497 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=13, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure 5f9d64419733%2C40969%2C1733268283535.1733268285003}] 2024-12-03T23:25:01,499 DEBUG [PEWorker-3 {}] master.SplitWALManager(158): Acquired split WAL worker=5f9d64419733,44411,1733268283680 2024-12-03T23:25:01,500 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=19, ppid=18, state=RUNNABLE, hasLock=false; SplitWALRemoteProcedure 5f9d64419733%2C40969%2C1733268283535.1733268285003, worker=5f9d64419733,44411,1733268283680}] 2024-12-03T23:25:01,652 DEBUG [RSProcedureDispatcher-pool-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-03T23:25:01,653 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57017, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-03T23:25:01,654 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44411 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SplitWALCallable, pid=19 2024-12-03T23:25:01,671 INFO [RS_LOG_REPLAY_OPS-regionserver/5f9d64419733:0-0 {event_type=RS_LOG_REPLAY, pid=19}] wal.WALSplitter(299): Splitting hdfs://localhost:35563/user/jenkins/test-data/e9061e95-7d9e-c12a-b652-9e8a1d87f0cf/WALs/5f9d64419733,40969,1733268283535-splitting/5f9d64419733%2C40969%2C1733268283535.1733268285003, size=1.6 K (1677bytes) 2024-12-03T23:25:01,671 INFO [RS_LOG_REPLAY_OPS-regionserver/5f9d64419733:0-0 {event_type=RS_LOG_REPLAY, pid=19}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:35563/user/jenkins/test-data/e9061e95-7d9e-c12a-b652-9e8a1d87f0cf/WALs/5f9d64419733,40969,1733268283535-splitting/5f9d64419733%2C40969%2C1733268283535.1733268285003 2024-12-03T23:25:01,671 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T23:25:01,671 INFO [RS_LOG_REPLAY_OPS-regionserver/5f9d64419733:0-0 {event_type=RS_LOG_REPLAY, pid=19}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:35563/user/jenkins/test-data/e9061e95-7d9e-c12a-b652-9e8a1d87f0cf/WALs/5f9d64419733,40969,1733268283535-splitting/5f9d64419733%2C40969%2C1733268283535.1733268285003 after 0ms 2024-12-03T23:25:01,672 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5f9d64419733,44215,1733268283642, seqNum=-1] 2024-12-03T23:25:01,674 DEBUG [RS_LOG_REPLAY_OPS-regionserver/5f9d64419733:0-0 {event_type=RS_LOG_REPLAY, pid=19}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:35563/user/jenkins/test-data/e9061e95-7d9e-c12a-b652-9e8a1d87f0cf/WALs/5f9d64419733,40969,1733268283535-splitting/5f9d64419733%2C40969%2C1733268283535.1733268285003: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-03T23:25:01,674 INFO [RS_LOG_REPLAY_OPS-regionserver/5f9d64419733:0-0 {event_type=RS_LOG_REPLAY, pid=19}] wal.WALSplitter(310): Open hdfs://localhost:35563/user/jenkins/test-data/e9061e95-7d9e-c12a-b652-9e8a1d87f0cf/WALs/5f9d64419733,40969,1733268283535-splitting/5f9d64419733%2C40969%2C1733268283535.1733268285003 took 4ms 2024-12-03T23:25:01,677 DEBUG [RS_LOG_REPLAY_OPS-regionserver/5f9d64419733:0-0 {event_type=RS_LOG_REPLAY, pid=19}] wal.WALSplitter(352): Last flushed sequenceid for 1745e7d6d10a6c6039d4c33aadf95a2f: last_flushed_sequence_id: 12 store_sequence_id { family_name: "cf1" sequence_id: 12 } store_sequence_id { family_name: "cf2" sequence_id: 12 } 2024-12-03T23:25:01,677 DEBUG [RS_LOG_REPLAY_OPS-regionserver/5f9d64419733:0-0 {event_type=RS_LOG_REPLAY, pid=19}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:35563/user/jenkins/test-data/e9061e95-7d9e-c12a-b652-9e8a1d87f0cf/WALs/5f9d64419733,40969,1733268283535-splitting/5f9d64419733%2C40969%2C1733268283535.1733268285003 so closing down 2024-12-03T23:25:01,677 DEBUG [RS_LOG_REPLAY_OPS-regionserver/5f9d64419733:0-0 {event_type=RS_LOG_REPLAY, pid=19}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-03T23:25:01,677 INFO [RS_LOG_REPLAY_OPS-regionserver/5f9d64419733:0-0 {event_type=RS_LOG_REPLAY, pid=19}] wal.OutputSink(145): 3 split writer threads finished 2024-12-03T23:25:01,677 INFO [RS_LOG_REPLAY_OPS-regionserver/5f9d64419733:0-0 {event_type=RS_LOG_REPLAY, pid=19}] wal.WALSplitter(425): Processed 6 edits across 0 Regions in 2 ms; skipped=6; WAL=hdfs://localhost:35563/user/jenkins/test-data/e9061e95-7d9e-c12a-b652-9e8a1d87f0cf/WALs/5f9d64419733,40969,1733268283535-splitting/5f9d64419733%2C40969%2C1733268283535.1733268285003, size=1.6 K, length=1677, corrupted=false, cancelled=false 2024-12-03T23:25:01,677 DEBUG [RS_LOG_REPLAY_OPS-regionserver/5f9d64419733:0-0 {event_type=RS_LOG_REPLAY, pid=19}] wal.WALSplitter(428): Completed split of hdfs://localhost:35563/user/jenkins/test-data/e9061e95-7d9e-c12a-b652-9e8a1d87f0cf/WALs/5f9d64419733,40969,1733268283535-splitting/5f9d64419733%2C40969%2C1733268283535.1733268285003, journal: Splitting hdfs://localhost:35563/user/jenkins/test-data/e9061e95-7d9e-c12a-b652-9e8a1d87f0cf/WALs/5f9d64419733,40969,1733268283535-splitting/5f9d64419733%2C40969%2C1733268283535.1733268285003, size=1.6 K (1677bytes) at 1733268301671Finishing writing output for hdfs://localhost:35563/user/jenkins/test-data/e9061e95-7d9e-c12a-b652-9e8a1d87f0cf/WALs/5f9d64419733,40969,1733268283535-splitting/5f9d64419733%2C40969%2C1733268283535.1733268285003 so closing down at 1733268301677 (+6 ms)3 split writer threads finished at 1733268301677Processed 6 edits across 0 Regions in 2 ms; skipped=6; WAL=hdfs://localhost:35563/user/jenkins/test-data/e9061e95-7d9e-c12a-b652-9e8a1d87f0cf/WALs/5f9d64419733,40969,1733268283535-splitting/5f9d64419733%2C40969%2C1733268283535.1733268285003, size=1.6 K, length=1677, corrupted=false, cancelled=false at 1733268301677 2024-12-03T23:25:01,677 DEBUG [RS_LOG_REPLAY_OPS-regionserver/5f9d64419733:0-0 {event_type=RS_LOG_REPLAY, pid=19}] regionserver.SplitLogWorker(218): Done splitting WAL hdfs://localhost:35563/user/jenkins/test-data/e9061e95-7d9e-c12a-b652-9e8a1d87f0cf/WALs/5f9d64419733,40969,1733268283535-splitting/5f9d64419733%2C40969%2C1733268283535.1733268285003 2024-12-03T23:25:01,677 DEBUG [RS_LOG_REPLAY_OPS-regionserver/5f9d64419733:0-0 {event_type=RS_LOG_REPLAY, pid=19}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=19 2024-12-03T23:25:01,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41995 {}] master.HMaster(4169): Remote procedure done, pid=19 2024-12-03T23:25:01,684 INFO [PEWorker-5 {}] wal.WALSplitUtil(143): Moved hdfs://localhost:35563/user/jenkins/test-data/e9061e95-7d9e-c12a-b652-9e8a1d87f0cf/WALs/5f9d64419733,40969,1733268283535-splitting/5f9d64419733%2C40969%2C1733268283535.1733268285003 to hdfs://localhost:35563/user/jenkins/test-data/e9061e95-7d9e-c12a-b652-9e8a1d87f0cf/oldWALs 2024-12-03T23:25:01,687 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=19, resume processing ppid=18 2024-12-03T23:25:01,687 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, ppid=18, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 5f9d64419733%2C40969%2C1733268283535.1733268285003, worker=5f9d64419733,44411,1733268283680 in 184 msec 2024-12-03T23:25:01,688 DEBUG [PEWorker-1 {}] master.SplitWALManager(172): Release split WAL worker=5f9d64419733,44411,1733268283680 2024-12-03T23:25:01,691 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=13 2024-12-03T23:25:01,691 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=13, state=SUCCESS, hasLock=false; SplitWALProcedure 5f9d64419733%2C40969%2C1733268283535.1733268285003, worker=5f9d64419733,44411,1733268283680 in 191 msec 2024-12-03T23:25:01,692 INFO [PEWorker-2 {}] master.SplitLogManager(171): hdfs://localhost:35563/user/jenkins/test-data/e9061e95-7d9e-c12a-b652-9e8a1d87f0cf/WALs/5f9d64419733,40969,1733268283535-splitting dir is empty, no logs to split. 2024-12-03T23:25:01,693 INFO [PEWorker-2 {}] master.SplitWALManager(105): 5f9d64419733,40969,1733268283535 WAL count=0, meta=false 2024-12-03T23:25:01,693 DEBUG [PEWorker-2 {}] procedure.ServerCrashProcedure(329): Check if 5f9d64419733,40969,1733268283535 WAL splitting is done? wals=0, meta=false 2024-12-03T23:25:01,695 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=1745e7d6d10a6c6039d4c33aadf95a2f, ASSIGN}] 2024-12-03T23:25:01,696 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=20, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=1745e7d6d10a6c6039d4c33aadf95a2f, ASSIGN 2024-12-03T23:25:01,697 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=20, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=1745e7d6d10a6c6039d4c33aadf95a2f, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-12-03T23:25:01,791 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.AbstractRpcClient(357): Not trying to connect to 5f9d64419733:40969 this server is in the failed servers list 2024-12-03T23:25:01,792 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733268298272.1745e7d6d10a6c6039d4c33aadf95a2f., hostname=5f9d64419733,40969,1733268283535, seqNum=12 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1733268298272.1745e7d6d10a6c6039d4c33aadf95a2f., hostname=5f9d64419733,40969,1733268283535, seqNum=12, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=5f9d64419733:40969 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 5f9d64419733:40969 2024-12-03T23:25:01,792 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733268298272.1745e7d6d10a6c6039d4c33aadf95a2f., hostname=5f9d64419733,40969,1733268283535, seqNum=12 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 5f9d64419733:40969 2024-12-03T23:25:01,792 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testReplayEditsAfterRegionMovedWithMultiCF,,1733268298272.1745e7d6d10a6c6039d4c33aadf95a2f., hostname=5f9d64419733,40969,1733268283535, seqNum=12 from cache 2024-12-03T23:25:01,847 DEBUG [5f9d64419733:41995 {}] balancer.BalancerClusterState(204): Hosts are {5f9d64419733=0} racks are {/default-rack=0} 2024-12-03T23:25:01,848 DEBUG [5f9d64419733:41995 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-12-03T23:25:01,848 DEBUG [5f9d64419733:41995 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-12-03T23:25:01,848 DEBUG [5f9d64419733:41995 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-03T23:25:01,848 DEBUG [5f9d64419733:41995 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-03T23:25:01,848 INFO [5f9d64419733:41995 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-03T23:25:01,848 INFO [5f9d64419733:41995 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-03T23:25:01,848 DEBUG [5f9d64419733:41995 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-12-03T23:25:01,848 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=1745e7d6d10a6c6039d4c33aadf95a2f, regionState=OPENING, regionLocation=5f9d64419733,44411,1733268283680 2024-12-03T23:25:01,849 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 5f9d64419733:40969 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 5f9d64419733/172.17.0.2:40969 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T23:25:01,850 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=5f9d64419733,40969,1733268283535, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=5f9d64419733,40969,1733268283535, seqNum=-1, error=java.net.ConnectException: Call to address=5f9d64419733:40969 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 5f9d64419733/172.17.0.2:40969 2024-12-03T23:25:01,850 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=5f9d64419733,40969,1733268283535, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-12-03T23:25:01,850 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=5f9d64419733,40969,1733268283535, seqNum=-1 from cache 2024-12-03T23:25:01,850 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address 5f9d64419733:40969 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 5f9d64419733/172.17.0.2:40969 2024-12-03T23:25:01,961 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T23:25:01,961 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5f9d64419733,44215,1733268283642, seqNum=-1] 2024-12-03T23:25:01,961 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T23:25:01,963 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49459, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T23:25:01,965 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=20, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=1745e7d6d10a6c6039d4c33aadf95a2f, ASSIGN because future has completed 2024-12-03T23:25:01,965 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=21, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1745e7d6d10a6c6039d4c33aadf95a2f, server=5f9d64419733,44411,1733268283680}] 2024-12-03T23:25:02,009 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1733268298272.1745e7d6d10a6c6039d4c33aadf95a2f., hostname=5f9d64419733,40969,1733268283535, seqNum=18] 2024-12-03T23:25:02,009 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] ipc.AbstractRpcClient(357): Not trying to connect to 5f9d64419733:40969 this server is in the failed servers list 2024-12-03T23:25:02,010 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733268298272.1745e7d6d10a6c6039d4c33aadf95a2f., hostname=5f9d64419733,40969,1733268283535, seqNum=18 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1733268298272.1745e7d6d10a6c6039d4c33aadf95a2f., hostname=5f9d64419733,40969,1733268283535, seqNum=18, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=5f9d64419733:40969 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 5f9d64419733:40969 2024-12-03T23:25:02,010 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733268298272.1745e7d6d10a6c6039d4c33aadf95a2f., hostname=5f9d64419733,40969,1733268283535, seqNum=18 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 5f9d64419733:40969 2024-12-03T23:25:02,010 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testReplayEditsAfterRegionMovedWithMultiCF,,1733268298272.1745e7d6d10a6c6039d4c33aadf95a2f., hostname=5f9d64419733,40969,1733268283535, seqNum=18 from cache 2024-12-03T23:25:02,122 INFO [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1733268298272.1745e7d6d10a6c6039d4c33aadf95a2f. 2024-12-03T23:25:02,122 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7752): Opening region: {ENCODED => 1745e7d6d10a6c6039d4c33aadf95a2f, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1733268298272.1745e7d6d10a6c6039d4c33aadf95a2f.', STARTKEY => '', ENDKEY => ''} 2024-12-03T23:25:02,123 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF 1745e7d6d10a6c6039d4c33aadf95a2f 2024-12-03T23:25:02,123 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1733268298272.1745e7d6d10a6c6039d4c33aadf95a2f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T23:25:02,123 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7794): checking encryption for 1745e7d6d10a6c6039d4c33aadf95a2f 2024-12-03T23:25:02,123 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7797): checking classloading for 1745e7d6d10a6c6039d4c33aadf95a2f 2024-12-03T23:25:02,125 INFO [StoreOpener-1745e7d6d10a6c6039d4c33aadf95a2f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 1745e7d6d10a6c6039d4c33aadf95a2f 2024-12-03T23:25:02,125 INFO [StoreOpener-1745e7d6d10a6c6039d4c33aadf95a2f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1745e7d6d10a6c6039d4c33aadf95a2f columnFamilyName cf1 2024-12-03T23:25:02,125 DEBUG [StoreOpener-1745e7d6d10a6c6039d4c33aadf95a2f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T23:25:02,132 DEBUG [StoreOpener-1745e7d6d10a6c6039d4c33aadf95a2f-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:35563/user/jenkins/test-data/e9061e95-7d9e-c12a-b652-9e8a1d87f0cf/data/default/testReplayEditsAfterRegionMovedWithMultiCF/1745e7d6d10a6c6039d4c33aadf95a2f/cf1/7d744d5820004695a930452e689f1b2a 2024-12-03T23:25:02,132 INFO [StoreOpener-1745e7d6d10a6c6039d4c33aadf95a2f-1 {}] regionserver.HStore(327): Store=1745e7d6d10a6c6039d4c33aadf95a2f/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T23:25:02,133 INFO [StoreOpener-1745e7d6d10a6c6039d4c33aadf95a2f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 1745e7d6d10a6c6039d4c33aadf95a2f 2024-12-03T23:25:02,133 INFO [StoreOpener-1745e7d6d10a6c6039d4c33aadf95a2f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1745e7d6d10a6c6039d4c33aadf95a2f columnFamilyName cf2 2024-12-03T23:25:02,134 DEBUG [StoreOpener-1745e7d6d10a6c6039d4c33aadf95a2f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T23:25:02,140 DEBUG [StoreOpener-1745e7d6d10a6c6039d4c33aadf95a2f-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:35563/user/jenkins/test-data/e9061e95-7d9e-c12a-b652-9e8a1d87f0cf/data/default/testReplayEditsAfterRegionMovedWithMultiCF/1745e7d6d10a6c6039d4c33aadf95a2f/cf2/fa67e9af7de2471d97906c12a08b2665 2024-12-03T23:25:02,140 INFO [StoreOpener-1745e7d6d10a6c6039d4c33aadf95a2f-1 {}] regionserver.HStore(327): Store=1745e7d6d10a6c6039d4c33aadf95a2f/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T23:25:02,140 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1038): replaying wal for 1745e7d6d10a6c6039d4c33aadf95a2f 2024-12-03T23:25:02,141 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35563/user/jenkins/test-data/e9061e95-7d9e-c12a-b652-9e8a1d87f0cf/data/default/testReplayEditsAfterRegionMovedWithMultiCF/1745e7d6d10a6c6039d4c33aadf95a2f 2024-12-03T23:25:02,142 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35563/user/jenkins/test-data/e9061e95-7d9e-c12a-b652-9e8a1d87f0cf/data/default/testReplayEditsAfterRegionMovedWithMultiCF/1745e7d6d10a6c6039d4c33aadf95a2f 2024-12-03T23:25:02,143 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1048): stopping wal replay for 1745e7d6d10a6c6039d4c33aadf95a2f 2024-12-03T23:25:02,143 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1060): Cleaning up temporary data for 1745e7d6d10a6c6039d4c33aadf95a2f 2024-12-03T23:25:02,144 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-12-03T23:25:02,145 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1093): writing seq id for 1745e7d6d10a6c6039d4c33aadf95a2f 2024-12-03T23:25:02,146 INFO [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1114): Opened 1745e7d6d10a6c6039d4c33aadf95a2f; next sequenceid=18; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59463779, jitterRate=-0.11392064392566681}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-12-03T23:25:02,146 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1745e7d6d10a6c6039d4c33aadf95a2f 2024-12-03T23:25:02,147 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1006): Region open journal for 1745e7d6d10a6c6039d4c33aadf95a2f: Running coprocessor pre-open hook at 1733268302123Writing region info on filesystem at 1733268302123Initializing all the Stores at 1733268302124 (+1 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733268302124Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733268302124Cleaning up temporary data from old regions at 1733268302143 (+19 ms)Running coprocessor post-open hooks at 1733268302146 (+3 ms)Region opened successfully at 1733268302147 (+1 ms) 2024-12-03T23:25:02,148 INFO [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1733268298272.1745e7d6d10a6c6039d4c33aadf95a2f., pid=21, masterSystemTime=1733268302117 2024-12-03T23:25:02,150 DEBUG [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1733268298272.1745e7d6d10a6c6039d4c33aadf95a2f. 2024-12-03T23:25:02,150 INFO [RS_OPEN_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1733268298272.1745e7d6d10a6c6039d4c33aadf95a2f. 2024-12-03T23:25:02,151 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=1745e7d6d10a6c6039d4c33aadf95a2f, regionState=OPEN, openSeqNum=18, regionLocation=5f9d64419733,44411,1733268283680 2024-12-03T23:25:02,153 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=21, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1745e7d6d10a6c6039d4c33aadf95a2f, server=5f9d64419733,44411,1733268283680 because future has completed 2024-12-03T23:25:02,157 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=21, resume processing ppid=20 2024-12-03T23:25:02,157 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, ppid=20, state=SUCCESS, hasLock=false; OpenRegionProcedure 1745e7d6d10a6c6039d4c33aadf95a2f, server=5f9d64419733,44411,1733268283680 in 189 msec 2024-12-03T23:25:02,160 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=13 2024-12-03T23:25:02,160 INFO [PEWorker-5 {}] procedure.ServerCrashProcedure(291): removed crashed server 5f9d64419733,40969,1733268283535 after splitting done 2024-12-03T23:25:02,160 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=13, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=1745e7d6d10a6c6039d4c33aadf95a2f, ASSIGN in 462 msec 2024-12-03T23:25:02,162 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; ServerCrashProcedure 5f9d64419733,40969,1733268283535, splitWal=true, meta=true in 1.4120 sec 2024-12-03T23:25:02,326 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1733268298272.1745e7d6d10a6c6039d4c33aadf95a2f., hostname=5f9d64419733,44411,1733268283680, seqNum=18] 2024-12-03T23:25:02,327 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T23:25:02,329 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60788, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T23:25:02,343 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayCompressed#testReplayEditsAfterRegionMovedWithMultiCF Thread=410 (was 413), OpenFileDescriptor=1048 (was 1022) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=222 (was 222), ProcessCount=11 (was 11), AvailableMemoryMB=7926 (was 7967) 2024-12-03T23:25:02,345 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1048 is superior to 1024 2024-12-03T23:25:02,357 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayCompressed#testReplayEditsAfterPartialFlush Thread=410, OpenFileDescriptor=1048, MaxFileDescriptor=1048576, SystemLoadAverage=222, ProcessCount=11, AvailableMemoryMB=7925 2024-12-03T23:25:02,357 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1048 is superior to 1024 2024-12-03T23:25:02,373 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-03T23:25:02,375 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-03T23:25:02,375 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-03T23:25:02,378 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-86428410, suffix=, logDir=hdfs://localhost:35563/hbase/WALs/hregion-86428410, archiveDir=hdfs://localhost:35563/hbase/oldWALs, maxLogs=32 2024-12-03T23:25:02,391 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-86428410/hregion-86428410.1733268302378, exclude list is [], retry=0 2024-12-03T23:25:02,394 DEBUG [AsyncFSWAL-20-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41487,DS-459d168f-61f7-403b-9139-91eca369c54e,DISK] 2024-12-03T23:25:02,395 DEBUG [AsyncFSWAL-20-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39829,DS-e6270720-ef81-458e-b5d6-959d16c4c378,DISK] 2024-12-03T23:25:02,395 DEBUG [AsyncFSWAL-20-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45491,DS-d55e5535-c0ce-4dc2-a45c-8e70eb010e56,DISK] 2024-12-03T23:25:02,398 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-86428410/hregion-86428410.1733268302378 2024-12-03T23:25:02,399 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36361:36361),(127.0.0.1/127.0.0.1:34451:34451),(127.0.0.1/127.0.0.1:38065:38065)] 2024-12-03T23:25:02,399 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 618ada30cbe30638741f57374b6f42f4, NAME => 'testReplayEditsWrittenViaHRegion,,1733268302373.618ada30cbe30638741f57374b6f42f4.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenViaHRegion', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35563/hbase 2024-12-03T23:25:02,409 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45491 is added to blk_1073741898_1076 (size=67) 2024-12-03T23:25:02,409 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41487 is added to blk_1073741898_1076 (size=67) 2024-12-03T23:25:02,410 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39829 is added to blk_1073741898_1076 (size=67) 2024-12-03T23:25:02,411 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1733268302373.618ada30cbe30638741f57374b6f42f4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T23:25:02,412 INFO [StoreOpener-618ada30cbe30638741f57374b6f42f4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 618ada30cbe30638741f57374b6f42f4 2024-12-03T23:25:02,413 INFO [StoreOpener-618ada30cbe30638741f57374b6f42f4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 618ada30cbe30638741f57374b6f42f4 columnFamilyName a 2024-12-03T23:25:02,413 DEBUG [StoreOpener-618ada30cbe30638741f57374b6f42f4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T23:25:02,414 INFO [StoreOpener-618ada30cbe30638741f57374b6f42f4-1 {}] regionserver.HStore(327): Store=618ada30cbe30638741f57374b6f42f4/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T23:25:02,414 INFO [StoreOpener-618ada30cbe30638741f57374b6f42f4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 618ada30cbe30638741f57374b6f42f4 2024-12-03T23:25:02,416 INFO [StoreOpener-618ada30cbe30638741f57374b6f42f4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 618ada30cbe30638741f57374b6f42f4 columnFamilyName b 2024-12-03T23:25:02,416 DEBUG [StoreOpener-618ada30cbe30638741f57374b6f42f4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T23:25:02,416 INFO [StoreOpener-618ada30cbe30638741f57374b6f42f4-1 {}] regionserver.HStore(327): Store=618ada30cbe30638741f57374b6f42f4/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T23:25:02,416 INFO [StoreOpener-618ada30cbe30638741f57374b6f42f4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 618ada30cbe30638741f57374b6f42f4 2024-12-03T23:25:02,417 INFO [StoreOpener-618ada30cbe30638741f57374b6f42f4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 618ada30cbe30638741f57374b6f42f4 columnFamilyName c 2024-12-03T23:25:02,417 DEBUG [StoreOpener-618ada30cbe30638741f57374b6f42f4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T23:25:02,418 INFO [StoreOpener-618ada30cbe30638741f57374b6f42f4-1 {}] regionserver.HStore(327): Store=618ada30cbe30638741f57374b6f42f4/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T23:25:02,418 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 618ada30cbe30638741f57374b6f42f4 2024-12-03T23:25:02,419 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35563/hbase/data/default/testReplayEditsWrittenViaHRegion/618ada30cbe30638741f57374b6f42f4 2024-12-03T23:25:02,419 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35563/hbase/data/default/testReplayEditsWrittenViaHRegion/618ada30cbe30638741f57374b6f42f4 2024-12-03T23:25:02,420 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 618ada30cbe30638741f57374b6f42f4 2024-12-03T23:25:02,420 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 618ada30cbe30638741f57374b6f42f4 2024-12-03T23:25:02,420 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-03T23:25:02,421 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 618ada30cbe30638741f57374b6f42f4 2024-12-03T23:25:02,424 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35563/hbase/data/default/testReplayEditsWrittenViaHRegion/618ada30cbe30638741f57374b6f42f4/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T23:25:02,424 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 618ada30cbe30638741f57374b6f42f4; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66911428, jitterRate=-0.002942025661468506}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-03T23:25:02,425 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 618ada30cbe30638741f57374b6f42f4: Writing region info on filesystem at 1733268302411Initializing all the Stores at 1733268302412 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733268302412Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733268302412Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733268302412Cleaning up temporary data from old regions at 1733268302420 (+8 ms)Region opened successfully at 1733268302425 (+5 ms) 2024-12-03T23:25:02,425 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 618ada30cbe30638741f57374b6f42f4, disabling compactions & flushes 2024-12-03T23:25:02,425 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1733268302373.618ada30cbe30638741f57374b6f42f4. 2024-12-03T23:25:02,425 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1733268302373.618ada30cbe30638741f57374b6f42f4. 2024-12-03T23:25:02,425 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1733268302373.618ada30cbe30638741f57374b6f42f4. after waiting 0 ms 2024-12-03T23:25:02,425 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1733268302373.618ada30cbe30638741f57374b6f42f4. 2024-12-03T23:25:02,425 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1733268302373.618ada30cbe30638741f57374b6f42f4. 2024-12-03T23:25:02,425 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 618ada30cbe30638741f57374b6f42f4: Waiting for close lock at 1733268302425Disabling compacts and flushes for region at 1733268302425Disabling writes for close at 1733268302425Writing region close event to WAL at 1733268302425Closed at 1733268302425 2024-12-03T23:25:02,428 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45491 is added to blk_1073741897_1075 (size=93) 2024-12-03T23:25:02,428 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39829 is added to blk_1073741897_1075 (size=93) 2024-12-03T23:25:02,428 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41487 is added to blk_1073741897_1075 (size=93) 2024-12-03T23:25:02,431 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-03T23:25:02,431 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-86428410:(num 1733268302378) 2024-12-03T23:25:02,431 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-03T23:25:02,433 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:35563/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733268302372, archiveDir=hdfs://localhost:35563/hbase/oldWALs, maxLogs=32 2024-12-03T23:25:02,447 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733268302372/wal.1733268302433, exclude list is [], retry=0 2024-12-03T23:25:02,449 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45491,DS-d55e5535-c0ce-4dc2-a45c-8e70eb010e56,DISK] 2024-12-03T23:25:02,450 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41487,DS-459d168f-61f7-403b-9139-91eca369c54e,DISK] 2024-12-03T23:25:02,450 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39829,DS-e6270720-ef81-458e-b5d6-959d16c4c378,DISK] 2024-12-03T23:25:02,452 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733268302372/wal.1733268302433 2024-12-03T23:25:02,452 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34451:34451),(127.0.0.1/127.0.0.1:36361:36361),(127.0.0.1/127.0.0.1:38065:38065)] 2024-12-03T23:25:02,452 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 618ada30cbe30638741f57374b6f42f4, NAME => 'testReplayEditsWrittenViaHRegion,,1733268302373.618ada30cbe30638741f57374b6f42f4.', STARTKEY => '', ENDKEY => ''} 2024-12-03T23:25:02,452 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1733268302373.618ada30cbe30638741f57374b6f42f4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T23:25:02,452 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 618ada30cbe30638741f57374b6f42f4 2024-12-03T23:25:02,452 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 618ada30cbe30638741f57374b6f42f4 2024-12-03T23:25:02,454 INFO [StoreOpener-618ada30cbe30638741f57374b6f42f4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 618ada30cbe30638741f57374b6f42f4 2024-12-03T23:25:02,455 INFO [StoreOpener-618ada30cbe30638741f57374b6f42f4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 618ada30cbe30638741f57374b6f42f4 columnFamilyName a 2024-12-03T23:25:02,455 DEBUG [StoreOpener-618ada30cbe30638741f57374b6f42f4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T23:25:02,455 INFO [StoreOpener-618ada30cbe30638741f57374b6f42f4-1 {}] regionserver.HStore(327): Store=618ada30cbe30638741f57374b6f42f4/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T23:25:02,456 INFO [StoreOpener-618ada30cbe30638741f57374b6f42f4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 618ada30cbe30638741f57374b6f42f4 2024-12-03T23:25:02,456 INFO [StoreOpener-618ada30cbe30638741f57374b6f42f4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 618ada30cbe30638741f57374b6f42f4 columnFamilyName b 2024-12-03T23:25:02,456 DEBUG [StoreOpener-618ada30cbe30638741f57374b6f42f4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T23:25:02,457 INFO [StoreOpener-618ada30cbe30638741f57374b6f42f4-1 {}] regionserver.HStore(327): Store=618ada30cbe30638741f57374b6f42f4/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T23:25:02,457 INFO [StoreOpener-618ada30cbe30638741f57374b6f42f4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 618ada30cbe30638741f57374b6f42f4 2024-12-03T23:25:02,457 INFO [StoreOpener-618ada30cbe30638741f57374b6f42f4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 618ada30cbe30638741f57374b6f42f4 columnFamilyName c 2024-12-03T23:25:02,458 DEBUG [StoreOpener-618ada30cbe30638741f57374b6f42f4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T23:25:02,458 INFO [StoreOpener-618ada30cbe30638741f57374b6f42f4-1 {}] regionserver.HStore(327): Store=618ada30cbe30638741f57374b6f42f4/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T23:25:02,458 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 618ada30cbe30638741f57374b6f42f4 2024-12-03T23:25:02,459 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35563/hbase/data/default/testReplayEditsWrittenViaHRegion/618ada30cbe30638741f57374b6f42f4 2024-12-03T23:25:02,460 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35563/hbase/data/default/testReplayEditsWrittenViaHRegion/618ada30cbe30638741f57374b6f42f4 2024-12-03T23:25:02,461 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 618ada30cbe30638741f57374b6f42f4 2024-12-03T23:25:02,461 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 618ada30cbe30638741f57374b6f42f4 2024-12-03T23:25:02,461 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-03T23:25:02,462 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 618ada30cbe30638741f57374b6f42f4 2024-12-03T23:25:02,463 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 618ada30cbe30638741f57374b6f42f4; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62378206, jitterRate=-0.07049229741096497}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-03T23:25:02,464 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 618ada30cbe30638741f57374b6f42f4: Writing region info on filesystem at 1733268302453Initializing all the Stores at 1733268302453Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733268302453Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733268302454 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733268302454Cleaning up temporary data from old regions at 1733268302461 (+7 ms)Region opened successfully at 1733268302464 (+3 ms) 2024-12-03T23:25:02,487 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 618ada30cbe30638741f57374b6f42f4 3/3 column families, dataSize=2.55 KB heapSize=5.44 KB 2024-12-03T23:25:02,504 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35563/hbase/data/default/testReplayEditsWrittenViaHRegion/618ada30cbe30638741f57374b6f42f4/.tmp/a/d6c0cf01aa834666b88417b58e52efa2 is 91, key is testReplayEditsWrittenViaHRegion/a:x0/1733268302464/Put/seqid=0 2024-12-03T23:25:02,514 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41487 is added to blk_1073741900_1078 (size=5958) 2024-12-03T23:25:02,514 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45491 is added to blk_1073741900_1078 (size=5958) 2024-12-03T23:25:02,514 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39829 is added to blk_1073741900_1078 (size=5958) 2024-12-03T23:25:02,515 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:35563/hbase/data/default/testReplayEditsWrittenViaHRegion/618ada30cbe30638741f57374b6f42f4/.tmp/a/d6c0cf01aa834666b88417b58e52efa2 2024-12-03T23:25:02,543 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35563/hbase/data/default/testReplayEditsWrittenViaHRegion/618ada30cbe30638741f57374b6f42f4/.tmp/b/5cf5c795038a4a69b18c5219d255f20e is 91, key is testReplayEditsWrittenViaHRegion/b:x0/1733268302472/Put/seqid=0 2024-12-03T23:25:02,551 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45491 is added to blk_1073741901_1079 (size=5958) 2024-12-03T23:25:02,551 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41487 is added to blk_1073741901_1079 (size=5958) 2024-12-03T23:25:02,552 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39829 is added to blk_1073741901_1079 (size=5958) 2024-12-03T23:25:02,552 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:35563/hbase/data/default/testReplayEditsWrittenViaHRegion/618ada30cbe30638741f57374b6f42f4/.tmp/b/5cf5c795038a4a69b18c5219d255f20e 2024-12-03T23:25:02,575 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35563/hbase/data/default/testReplayEditsWrittenViaHRegion/618ada30cbe30638741f57374b6f42f4/.tmp/c/3ef8a1004cf24c2092ca0117047d9e73 is 91, key is testReplayEditsWrittenViaHRegion/c:x0/1733268302479/Put/seqid=0 2024-12-03T23:25:02,582 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45491 is added to blk_1073741902_1080 (size=5958) 2024-12-03T23:25:02,582 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39829 is added to blk_1073741902_1080 (size=5958) 2024-12-03T23:25:02,582 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41487 is added to blk_1073741902_1080 (size=5958) 2024-12-03T23:25:02,583 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:35563/hbase/data/default/testReplayEditsWrittenViaHRegion/618ada30cbe30638741f57374b6f42f4/.tmp/c/3ef8a1004cf24c2092ca0117047d9e73 2024-12-03T23:25:02,589 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35563/hbase/data/default/testReplayEditsWrittenViaHRegion/618ada30cbe30638741f57374b6f42f4/.tmp/a/d6c0cf01aa834666b88417b58e52efa2 as hdfs://localhost:35563/hbase/data/default/testReplayEditsWrittenViaHRegion/618ada30cbe30638741f57374b6f42f4/a/d6c0cf01aa834666b88417b58e52efa2 2024-12-03T23:25:02,594 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35563/hbase/data/default/testReplayEditsWrittenViaHRegion/618ada30cbe30638741f57374b6f42f4/a/d6c0cf01aa834666b88417b58e52efa2, entries=10, sequenceid=33, filesize=5.8 K 2024-12-03T23:25:02,595 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35563/hbase/data/default/testReplayEditsWrittenViaHRegion/618ada30cbe30638741f57374b6f42f4/.tmp/b/5cf5c795038a4a69b18c5219d255f20e as hdfs://localhost:35563/hbase/data/default/testReplayEditsWrittenViaHRegion/618ada30cbe30638741f57374b6f42f4/b/5cf5c795038a4a69b18c5219d255f20e 2024-12-03T23:25:02,601 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35563/hbase/data/default/testReplayEditsWrittenViaHRegion/618ada30cbe30638741f57374b6f42f4/b/5cf5c795038a4a69b18c5219d255f20e, entries=10, sequenceid=33, filesize=5.8 K 2024-12-03T23:25:02,602 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35563/hbase/data/default/testReplayEditsWrittenViaHRegion/618ada30cbe30638741f57374b6f42f4/.tmp/c/3ef8a1004cf24c2092ca0117047d9e73 as hdfs://localhost:35563/hbase/data/default/testReplayEditsWrittenViaHRegion/618ada30cbe30638741f57374b6f42f4/c/3ef8a1004cf24c2092ca0117047d9e73 2024-12-03T23:25:02,608 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35563/hbase/data/default/testReplayEditsWrittenViaHRegion/618ada30cbe30638741f57374b6f42f4/c/3ef8a1004cf24c2092ca0117047d9e73, entries=10, sequenceid=33, filesize=5.8 K 2024-12-03T23:25:02,610 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~2.55 KB/2610, heapSize ~5.39 KB/5520, currentSize=0 B/0 for 618ada30cbe30638741f57374b6f42f4 in 123ms, sequenceid=33, compaction requested=false 2024-12-03T23:25:02,610 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 618ada30cbe30638741f57374b6f42f4: 2024-12-03T23:25:02,610 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 618ada30cbe30638741f57374b6f42f4, disabling compactions & flushes 2024-12-03T23:25:02,610 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1733268302373.618ada30cbe30638741f57374b6f42f4. 2024-12-03T23:25:02,610 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1733268302373.618ada30cbe30638741f57374b6f42f4. 2024-12-03T23:25:02,610 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1733268302373.618ada30cbe30638741f57374b6f42f4. after waiting 0 ms 2024-12-03T23:25:02,610 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1733268302373.618ada30cbe30638741f57374b6f42f4. 2024-12-03T23:25:02,612 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1733268302373.618ada30cbe30638741f57374b6f42f4. 2024-12-03T23:25:02,612 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 618ada30cbe30638741f57374b6f42f4: Waiting for close lock at 1733268302610Disabling compacts and flushes for region at 1733268302610Disabling writes for close at 1733268302610Writing region close event to WAL at 1733268302611 (+1 ms)Closed at 1733268302612 (+1 ms) 2024-12-03T23:25:02,614 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45491 is added to blk_1073741899_1077 (size=2873) 2024-12-03T23:25:02,614 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39829 is added to blk_1073741899_1077 (size=2873) 2024-12-03T23:25:02,614 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41487 is added to blk_1073741899_1077 (size=2873) 2024-12-03T23:25:02,620 DEBUG [Time-limited test {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35563/hbase/data/default/testReplayEditsWrittenViaHRegion/618ada30cbe30638741f57374b6f42f4/b/5cf5c795038a4a69b18c5219d255f20e to hdfs://localhost:35563/hbase/archive/data/default/testReplayEditsWrittenViaHRegion/618ada30cbe30638741f57374b6f42f4/b/5cf5c795038a4a69b18c5219d255f20e 2024-12-03T23:25:02,637 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:35563/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733268302372/wal.1733268302433, size=2.8 K (2873bytes) 2024-12-03T23:25:02,637 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:35563/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733268302372/wal.1733268302433 2024-12-03T23:25:02,637 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:35563/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733268302372/wal.1733268302433 after 0ms 2024-12-03T23:25:02,640 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:35563/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733268302372/wal.1733268302433: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-03T23:25:02,640 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:35563/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733268302372/wal.1733268302433 took 3ms 2024-12-03T23:25:02,642 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:35563/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733268302372/wal.1733268302433 so closing down 2024-12-03T23:25:02,642 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-03T23:25:02,643 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1733268302433.temp 2024-12-03T23:25:02,645 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:35563/hbase/data/default/testReplayEditsWrittenViaHRegion/618ada30cbe30638741f57374b6f42f4/recovered.edits/0000000000000000003-wal.1733268302433.temp 2024-12-03T23:25:02,645 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-03T23:25:02,651 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45491 is added to blk_1073741903_1081 (size=2312) 2024-12-03T23:25:02,651 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41487 is added to blk_1073741903_1081 (size=2312) 2024-12-03T23:25:02,652 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39829 is added to blk_1073741903_1081 (size=2312) 2024-12-03T23:25:02,652 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:35563/hbase/data/default/testReplayEditsWrittenViaHRegion/618ada30cbe30638741f57374b6f42f4/recovered.edits/0000000000000000003-wal.1733268302433.temp (wrote 30 edits, skipped 0 edits in 0 ms) 2024-12-03T23:25:02,654 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:35563/hbase/data/default/testReplayEditsWrittenViaHRegion/618ada30cbe30638741f57374b6f42f4/recovered.edits/0000000000000000003-wal.1733268302433.temp to hdfs://localhost:35563/hbase/data/default/testReplayEditsWrittenViaHRegion/618ada30cbe30638741f57374b6f42f4/recovered.edits/0000000000000000032 2024-12-03T23:25:02,654 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 32 edits across 1 Regions in 14 ms; skipped=2; WAL=hdfs://localhost:35563/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733268302372/wal.1733268302433, size=2.8 K, length=2873, corrupted=false, cancelled=false 2024-12-03T23:25:02,654 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:35563/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733268302372/wal.1733268302433, journal: Splitting hdfs://localhost:35563/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733268302372/wal.1733268302433, size=2.8 K (2873bytes) at 1733268302637Finishing writing output for hdfs://localhost:35563/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733268302372/wal.1733268302433 so closing down at 1733268302642 (+5 ms)Creating recovered edits writer path=hdfs://localhost:35563/hbase/data/default/testReplayEditsWrittenViaHRegion/618ada30cbe30638741f57374b6f42f4/recovered.edits/0000000000000000003-wal.1733268302433.temp at 1733268302645 (+3 ms)3 split writer threads finished at 1733268302645Closed recovered edits writer path=hdfs://localhost:35563/hbase/data/default/testReplayEditsWrittenViaHRegion/618ada30cbe30638741f57374b6f42f4/recovered.edits/0000000000000000003-wal.1733268302433.temp (wrote 30 edits, skipped 0 edits in 0 ms) at 1733268302652 (+7 ms)Rename recovered edits hdfs://localhost:35563/hbase/data/default/testReplayEditsWrittenViaHRegion/618ada30cbe30638741f57374b6f42f4/recovered.edits/0000000000000000003-wal.1733268302433.temp to hdfs://localhost:35563/hbase/data/default/testReplayEditsWrittenViaHRegion/618ada30cbe30638741f57374b6f42f4/recovered.edits/0000000000000000032 at 1733268302654 (+2 ms)Processed 32 edits across 1 Regions in 14 ms; skipped=2; WAL=hdfs://localhost:35563/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733268302372/wal.1733268302433, size=2.8 K, length=2873, corrupted=false, cancelled=false at 1733268302654 2024-12-03T23:25:02,656 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:35563/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733268302372/wal.1733268302433 to hdfs://localhost:35563/hbase/oldWALs/wal.1733268302433 2024-12-03T23:25:02,656 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:35563/hbase/data/default/testReplayEditsWrittenViaHRegion/618ada30cbe30638741f57374b6f42f4/recovered.edits/0000000000000000032 2024-12-03T23:25:02,656 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-03T23:25:02,658 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:35563/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733268302372, archiveDir=hdfs://localhost:35563/hbase/oldWALs, maxLogs=32 2024-12-03T23:25:02,672 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733268302372/wal.1733268302658, exclude list is [], retry=0 2024-12-03T23:25:02,675 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41487,DS-459d168f-61f7-403b-9139-91eca369c54e,DISK] 2024-12-03T23:25:02,676 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45491,DS-d55e5535-c0ce-4dc2-a45c-8e70eb010e56,DISK] 2024-12-03T23:25:02,676 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39829,DS-e6270720-ef81-458e-b5d6-959d16c4c378,DISK] 2024-12-03T23:25:02,679 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733268302372/wal.1733268302658 2024-12-03T23:25:02,679 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36361:36361),(127.0.0.1/127.0.0.1:34451:34451),(127.0.0.1/127.0.0.1:38065:38065)] 2024-12-03T23:25:02,679 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 618ada30cbe30638741f57374b6f42f4, NAME => 'testReplayEditsWrittenViaHRegion,,1733268302373.618ada30cbe30638741f57374b6f42f4.', STARTKEY => '', ENDKEY => ''} 2024-12-03T23:25:02,680 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1733268302373.618ada30cbe30638741f57374b6f42f4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T23:25:02,680 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 618ada30cbe30638741f57374b6f42f4 2024-12-03T23:25:02,680 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 618ada30cbe30638741f57374b6f42f4 2024-12-03T23:25:02,681 INFO [StoreOpener-618ada30cbe30638741f57374b6f42f4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 618ada30cbe30638741f57374b6f42f4 2024-12-03T23:25:02,682 INFO [StoreOpener-618ada30cbe30638741f57374b6f42f4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 618ada30cbe30638741f57374b6f42f4 columnFamilyName a 2024-12-03T23:25:02,682 DEBUG [StoreOpener-618ada30cbe30638741f57374b6f42f4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T23:25:02,688 DEBUG [StoreOpener-618ada30cbe30638741f57374b6f42f4-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:35563/hbase/data/default/testReplayEditsWrittenViaHRegion/618ada30cbe30638741f57374b6f42f4/a/d6c0cf01aa834666b88417b58e52efa2 2024-12-03T23:25:02,688 INFO [StoreOpener-618ada30cbe30638741f57374b6f42f4-1 {}] regionserver.HStore(327): Store=618ada30cbe30638741f57374b6f42f4/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T23:25:02,688 INFO [StoreOpener-618ada30cbe30638741f57374b6f42f4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 618ada30cbe30638741f57374b6f42f4 2024-12-03T23:25:02,689 INFO [StoreOpener-618ada30cbe30638741f57374b6f42f4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 618ada30cbe30638741f57374b6f42f4 columnFamilyName b 2024-12-03T23:25:02,689 DEBUG [StoreOpener-618ada30cbe30638741f57374b6f42f4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T23:25:02,689 INFO [StoreOpener-618ada30cbe30638741f57374b6f42f4-1 {}] regionserver.HStore(327): Store=618ada30cbe30638741f57374b6f42f4/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T23:25:02,690 INFO [StoreOpener-618ada30cbe30638741f57374b6f42f4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 618ada30cbe30638741f57374b6f42f4 2024-12-03T23:25:02,690 INFO [StoreOpener-618ada30cbe30638741f57374b6f42f4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 618ada30cbe30638741f57374b6f42f4 columnFamilyName c 2024-12-03T23:25:02,690 DEBUG [StoreOpener-618ada30cbe30638741f57374b6f42f4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T23:25:02,694 DEBUG [StoreOpener-618ada30cbe30638741f57374b6f42f4-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:35563/hbase/data/default/testReplayEditsWrittenViaHRegion/618ada30cbe30638741f57374b6f42f4/c/3ef8a1004cf24c2092ca0117047d9e73 2024-12-03T23:25:02,695 INFO [StoreOpener-618ada30cbe30638741f57374b6f42f4-1 {}] regionserver.HStore(327): Store=618ada30cbe30638741f57374b6f42f4/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T23:25:02,695 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 618ada30cbe30638741f57374b6f42f4 2024-12-03T23:25:02,695 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35563/hbase/data/default/testReplayEditsWrittenViaHRegion/618ada30cbe30638741f57374b6f42f4 2024-12-03T23:25:02,697 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:35563/hbase/data/default/testReplayEditsWrittenViaHRegion/618ada30cbe30638741f57374b6f42f4 2024-12-03T23:25:02,697 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:35563/hbase/data/default/testReplayEditsWrittenViaHRegion/618ada30cbe30638741f57374b6f42f4/recovered.edits/0000000000000000032 2024-12-03T23:25:02,699 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:35563/hbase/data/default/testReplayEditsWrittenViaHRegion/618ada30cbe30638741f57374b6f42f4/recovered.edits/0000000000000000032: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-03T23:25:02,700 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 10, skipped 20, firstSequenceIdInLog=3, maxSequenceIdInLog=32, path=hdfs://localhost:35563/hbase/data/default/testReplayEditsWrittenViaHRegion/618ada30cbe30638741f57374b6f42f4/recovered.edits/0000000000000000032 2024-12-03T23:25:02,700 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 618ada30cbe30638741f57374b6f42f4 3/3 column families, dataSize=870 B heapSize=2.31 KB 2024-12-03T23:25:02,715 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35563/hbase/data/default/testReplayEditsWrittenViaHRegion/618ada30cbe30638741f57374b6f42f4/.tmp/b/1507c9d80f55461d8f3c967f6db3fb9c is 91, key is testReplayEditsWrittenViaHRegion/b:x0/1733268302472/Put/seqid=0 2024-12-03T23:25:02,722 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45491 is added to blk_1073741905_1083 (size=5958) 2024-12-03T23:25:02,722 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39829 is added to blk_1073741905_1083 (size=5958) 2024-12-03T23:25:02,723 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41487 is added to blk_1073741905_1083 (size=5958) 2024-12-03T23:25:02,723 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=32 (bloomFilter=true), to=hdfs://localhost:35563/hbase/data/default/testReplayEditsWrittenViaHRegion/618ada30cbe30638741f57374b6f42f4/.tmp/b/1507c9d80f55461d8f3c967f6db3fb9c 2024-12-03T23:25:02,729 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35563/hbase/data/default/testReplayEditsWrittenViaHRegion/618ada30cbe30638741f57374b6f42f4/.tmp/b/1507c9d80f55461d8f3c967f6db3fb9c as hdfs://localhost:35563/hbase/data/default/testReplayEditsWrittenViaHRegion/618ada30cbe30638741f57374b6f42f4/b/1507c9d80f55461d8f3c967f6db3fb9c 2024-12-03T23:25:02,734 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35563/hbase/data/default/testReplayEditsWrittenViaHRegion/618ada30cbe30638741f57374b6f42f4/b/1507c9d80f55461d8f3c967f6db3fb9c, entries=10, sequenceid=32, filesize=5.8 K 2024-12-03T23:25:02,734 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~870 B/870, heapSize ~1.80 KB/1840, currentSize=0 B/0 for 618ada30cbe30638741f57374b6f42f4 in 34ms, sequenceid=32, compaction requested=false; wal=null 2024-12-03T23:25:02,735 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:35563/hbase/data/default/testReplayEditsWrittenViaHRegion/618ada30cbe30638741f57374b6f42f4/recovered.edits/0000000000000000032 2024-12-03T23:25:02,736 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 618ada30cbe30638741f57374b6f42f4 2024-12-03T23:25:02,736 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 618ada30cbe30638741f57374b6f42f4 2024-12-03T23:25:02,737 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-03T23:25:02,738 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 618ada30cbe30638741f57374b6f42f4 2024-12-03T23:25:02,740 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35563/hbase/data/default/testReplayEditsWrittenViaHRegion/618ada30cbe30638741f57374b6f42f4/recovered.edits/33.seqid, newMaxSeqId=33, maxSeqId=1 2024-12-03T23:25:02,741 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 618ada30cbe30638741f57374b6f42f4; next sequenceid=34; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74868231, jitterRate=0.1156235784292221}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-03T23:25:02,742 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 618ada30cbe30638741f57374b6f42f4: Writing region info on filesystem at 1733268302680Initializing all the Stores at 1733268302681 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733268302681Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733268302681Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733268302681Obtaining lock to block concurrent updates at 1733268302701 (+20 ms)Preparing flush snapshotting stores in 618ada30cbe30638741f57374b6f42f4 at 1733268302701Finished memstore snapshotting testReplayEditsWrittenViaHRegion,,1733268302373.618ada30cbe30638741f57374b6f42f4., syncing WAL and waiting on mvcc, flushsize=dataSize=870, getHeapSize=2320, getOffHeapSize=0, getCellsCount=10 at 1733268302701Flushing stores of testReplayEditsWrittenViaHRegion,,1733268302373.618ada30cbe30638741f57374b6f42f4. at 1733268302701Flushing 618ada30cbe30638741f57374b6f42f4/b: creating writer at 1733268302701Flushing 618ada30cbe30638741f57374b6f42f4/b: appending metadata at 1733268302715 (+14 ms)Flushing 618ada30cbe30638741f57374b6f42f4/b: closing flushed file at 1733268302715Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@15715247: reopening flushed file at 1733268302728 (+13 ms)Finished flush of dataSize ~870 B/870, heapSize ~1.80 KB/1840, currentSize=0 B/0 for 618ada30cbe30638741f57374b6f42f4 in 34ms, sequenceid=32, compaction requested=false; wal=null at 1733268302734 (+6 ms)Cleaning up temporary data from old regions at 1733268302736 (+2 ms)Region opened successfully at 1733268302742 (+6 ms) 2024-12-03T23:25:02,764 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayCompressed#testReplayEditsAfterPartialFlush Thread=419 (was 410) Potentially hanging thread: PacketResponder: BP-2057003906-172.17.0.2-1733268279817:blk_1073741904_1082, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-773788190_22 at /127.0.0.1:36630 [Receiving block BP-2057003906-172.17.0.2-1733268279817:blk_1073741904_1082] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-773788190_22 at /127.0.0.1:44022 [Waiting for operation #29] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-20-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-773788190_22 at /127.0.0.1:52316 [Waiting for operation #58] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-20-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-773788190_22 at /127.0.0.1:59602 [Receiving block BP-2057003906-172.17.0.2-1733268279817:blk_1073741904_1082] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-2057003906-172.17.0.2-1733268279817:blk_1073741904_1082, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-20-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-773788190_22 at /127.0.0.1:40880 [Waiting for operation #36] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-773788190_22 at /127.0.0.1:33686 [Receiving block BP-2057003906-172.17.0.2-1733268279817:blk_1073741904_1082] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-2057003906-172.17.0.2-1733268279817:blk_1073741904_1082, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1124 (was 1048) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=222 (was 222), ProcessCount=11 (was 11), AvailableMemoryMB=7919 (was 7925) 2024-12-03T23:25:02,764 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1124 is superior to 1024 2024-12-03T23:25:02,777 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayCompressed#testReplayEditsAfterAbortingFlush Thread=419, OpenFileDescriptor=1124, MaxFileDescriptor=1048576, SystemLoadAverage=222, ProcessCount=11, AvailableMemoryMB=7918 2024-12-03T23:25:02,777 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1124 is superior to 1024 2024-12-03T23:25:02,793 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-03T23:25:02,795 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-03T23:25:02,796 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-03T23:25:02,798 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-23310140, suffix=, logDir=hdfs://localhost:35563/hbase/WALs/hregion-23310140, archiveDir=hdfs://localhost:35563/hbase/oldWALs, maxLogs=32 2024-12-03T23:25:02,812 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-23310140/hregion-23310140.1733268302799, exclude list is [], retry=0 2024-12-03T23:25:02,815 DEBUG [AsyncFSWAL-22-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41487,DS-459d168f-61f7-403b-9139-91eca369c54e,DISK] 2024-12-03T23:25:02,815 DEBUG [AsyncFSWAL-22-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45491,DS-d55e5535-c0ce-4dc2-a45c-8e70eb010e56,DISK] 2024-12-03T23:25:02,816 DEBUG [AsyncFSWAL-22-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39829,DS-e6270720-ef81-458e-b5d6-959d16c4c378,DISK] 2024-12-03T23:25:02,819 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-23310140/hregion-23310140.1733268302799 2024-12-03T23:25:02,819 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36361:36361),(127.0.0.1/127.0.0.1:34451:34451),(127.0.0.1/127.0.0.1:38065:38065)] 2024-12-03T23:25:02,819 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 22ac440ef159085837f649a68330baac, NAME => 'testReplayEditsAfterAbortingFlush,,1733268302794.22ac440ef159085837f649a68330baac.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsAfterAbortingFlush', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35563/hbase 2024-12-03T23:25:02,827 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41487 is added to blk_1073741907_1085 (size=68) 2024-12-03T23:25:02,827 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39829 is added to blk_1073741907_1085 (size=68) 2024-12-03T23:25:02,828 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45491 is added to blk_1073741907_1085 (size=68) 2024-12-03T23:25:02,828 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterAbortingFlush,,1733268302794.22ac440ef159085837f649a68330baac.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T23:25:02,832 INFO [StoreOpener-22ac440ef159085837f649a68330baac-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 22ac440ef159085837f649a68330baac 2024-12-03T23:25:02,834 INFO [StoreOpener-22ac440ef159085837f649a68330baac-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 22ac440ef159085837f649a68330baac columnFamilyName a 2024-12-03T23:25:02,834 DEBUG [StoreOpener-22ac440ef159085837f649a68330baac-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T23:25:02,835 INFO [StoreOpener-22ac440ef159085837f649a68330baac-1 {}] regionserver.HStore(327): Store=22ac440ef159085837f649a68330baac/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T23:25:02,835 INFO [StoreOpener-22ac440ef159085837f649a68330baac-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 22ac440ef159085837f649a68330baac 2024-12-03T23:25:02,836 INFO [StoreOpener-22ac440ef159085837f649a68330baac-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 22ac440ef159085837f649a68330baac columnFamilyName b 2024-12-03T23:25:02,836 DEBUG [StoreOpener-22ac440ef159085837f649a68330baac-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T23:25:02,837 INFO [StoreOpener-22ac440ef159085837f649a68330baac-1 {}] regionserver.HStore(327): Store=22ac440ef159085837f649a68330baac/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T23:25:02,837 INFO [StoreOpener-22ac440ef159085837f649a68330baac-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 22ac440ef159085837f649a68330baac 2024-12-03T23:25:02,838 INFO [StoreOpener-22ac440ef159085837f649a68330baac-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 22ac440ef159085837f649a68330baac columnFamilyName c 2024-12-03T23:25:02,838 DEBUG [StoreOpener-22ac440ef159085837f649a68330baac-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T23:25:02,839 INFO [StoreOpener-22ac440ef159085837f649a68330baac-1 {}] regionserver.HStore(327): Store=22ac440ef159085837f649a68330baac/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T23:25:02,839 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 22ac440ef159085837f649a68330baac 2024-12-03T23:25:02,840 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35563/hbase/data/default/testReplayEditsAfterAbortingFlush/22ac440ef159085837f649a68330baac 2024-12-03T23:25:02,840 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35563/hbase/data/default/testReplayEditsAfterAbortingFlush/22ac440ef159085837f649a68330baac 2024-12-03T23:25:02,841 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 22ac440ef159085837f649a68330baac 2024-12-03T23:25:02,841 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 22ac440ef159085837f649a68330baac 2024-12-03T23:25:02,842 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterAbortingFlush descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-03T23:25:02,843 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 22ac440ef159085837f649a68330baac 2024-12-03T23:25:02,846 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35563/hbase/data/default/testReplayEditsAfterAbortingFlush/22ac440ef159085837f649a68330baac/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T23:25:02,846 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 22ac440ef159085837f649a68330baac; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60184258, jitterRate=-0.10318467020988464}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-03T23:25:02,847 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 22ac440ef159085837f649a68330baac: Writing region info on filesystem at 1733268302828Initializing all the Stores at 1733268302829 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733268302829Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733268302832 (+3 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733268302832Cleaning up temporary data from old regions at 1733268302841 (+9 ms)Region opened successfully at 1733268302847 (+6 ms) 2024-12-03T23:25:02,847 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 22ac440ef159085837f649a68330baac, disabling compactions & flushes 2024-12-03T23:25:02,847 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterAbortingFlush,,1733268302794.22ac440ef159085837f649a68330baac. 2024-12-03T23:25:02,847 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterAbortingFlush,,1733268302794.22ac440ef159085837f649a68330baac. 2024-12-03T23:25:02,847 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterAbortingFlush,,1733268302794.22ac440ef159085837f649a68330baac. after waiting 0 ms 2024-12-03T23:25:02,847 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterAbortingFlush,,1733268302794.22ac440ef159085837f649a68330baac. 2024-12-03T23:25:02,848 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsAfterAbortingFlush,,1733268302794.22ac440ef159085837f649a68330baac. 2024-12-03T23:25:02,848 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 22ac440ef159085837f649a68330baac: Waiting for close lock at 1733268302847Disabling compacts and flushes for region at 1733268302847Disabling writes for close at 1733268302847Writing region close event to WAL at 1733268302848 (+1 ms)Closed at 1733268302848 2024-12-03T23:25:02,851 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39829 is added to blk_1073741906_1084 (size=93) 2024-12-03T23:25:02,851 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45491 is added to blk_1073741906_1084 (size=93) 2024-12-03T23:25:02,851 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41487 is added to blk_1073741906_1084 (size=93) 2024-12-03T23:25:02,853 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-03T23:25:02,853 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-23310140:(num 1733268302799) 2024-12-03T23:25:02,853 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-03T23:25:02,855 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:35563/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733268302792, archiveDir=hdfs://localhost:35563/hbase/oldWALs, maxLogs=32 2024-12-03T23:25:02,869 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733268302792/wal.1733268302855, exclude list is [], retry=0 2024-12-03T23:25:02,872 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39829,DS-e6270720-ef81-458e-b5d6-959d16c4c378,DISK] 2024-12-03T23:25:02,872 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45491,DS-d55e5535-c0ce-4dc2-a45c-8e70eb010e56,DISK] 2024-12-03T23:25:02,873 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41487,DS-459d168f-61f7-403b-9139-91eca369c54e,DISK] 2024-12-03T23:25:02,874 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733268302792/wal.1733268302855 2024-12-03T23:25:02,874 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:38065:38065),(127.0.0.1/127.0.0.1:34451:34451),(127.0.0.1/127.0.0.1:36361:36361)] 2024-12-03T23:25:02,939 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 22ac440ef159085837f649a68330baac, NAME => 'testReplayEditsAfterAbortingFlush,,1733268302794.22ac440ef159085837f649a68330baac.', STARTKEY => '', ENDKEY => ''} 2024-12-03T23:25:02,942 DEBUG [Time-limited test {}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterAbortingFlush 22ac440ef159085837f649a68330baac 2024-12-03T23:25:02,942 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterAbortingFlush,,1733268302794.22ac440ef159085837f649a68330baac.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T23:25:02,942 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 22ac440ef159085837f649a68330baac 2024-12-03T23:25:02,942 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 22ac440ef159085837f649a68330baac 2024-12-03T23:25:02,944 INFO [StoreOpener-22ac440ef159085837f649a68330baac-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 22ac440ef159085837f649a68330baac 2024-12-03T23:25:02,945 INFO [StoreOpener-22ac440ef159085837f649a68330baac-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 22ac440ef159085837f649a68330baac columnFamilyName a 2024-12-03T23:25:02,945 DEBUG [StoreOpener-22ac440ef159085837f649a68330baac-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T23:25:02,946 INFO [StoreOpener-22ac440ef159085837f649a68330baac-1 {}] regionserver.HStore(327): Store=22ac440ef159085837f649a68330baac/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T23:25:02,946 INFO [StoreOpener-22ac440ef159085837f649a68330baac-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 22ac440ef159085837f649a68330baac 2024-12-03T23:25:02,947 INFO [StoreOpener-22ac440ef159085837f649a68330baac-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 22ac440ef159085837f649a68330baac columnFamilyName b 2024-12-03T23:25:02,947 DEBUG [StoreOpener-22ac440ef159085837f649a68330baac-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T23:25:02,947 INFO [StoreOpener-22ac440ef159085837f649a68330baac-1 {}] regionserver.HStore(327): Store=22ac440ef159085837f649a68330baac/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T23:25:02,947 INFO [StoreOpener-22ac440ef159085837f649a68330baac-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 22ac440ef159085837f649a68330baac 2024-12-03T23:25:02,948 INFO [StoreOpener-22ac440ef159085837f649a68330baac-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 22ac440ef159085837f649a68330baac columnFamilyName c 2024-12-03T23:25:02,948 DEBUG [StoreOpener-22ac440ef159085837f649a68330baac-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T23:25:02,948 INFO [StoreOpener-22ac440ef159085837f649a68330baac-1 {}] regionserver.HStore(327): Store=22ac440ef159085837f649a68330baac/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T23:25:02,949 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 22ac440ef159085837f649a68330baac 2024-12-03T23:25:02,949 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35563/hbase/data/default/testReplayEditsAfterAbortingFlush/22ac440ef159085837f649a68330baac 2024-12-03T23:25:02,950 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35563/hbase/data/default/testReplayEditsAfterAbortingFlush/22ac440ef159085837f649a68330baac 2024-12-03T23:25:02,951 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 22ac440ef159085837f649a68330baac 2024-12-03T23:25:02,951 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 22ac440ef159085837f649a68330baac 2024-12-03T23:25:02,952 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterAbortingFlush descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-03T23:25:02,953 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 22ac440ef159085837f649a68330baac 2024-12-03T23:25:02,954 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 22ac440ef159085837f649a68330baac; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73634642, jitterRate=0.09724166989326477}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-03T23:25:02,954 DEBUG [Time-limited test {}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 22ac440ef159085837f649a68330baac 2024-12-03T23:25:02,955 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 22ac440ef159085837f649a68330baac: Running coprocessor pre-open hook at 1733268302942Writing region info on filesystem at 1733268302942Initializing all the Stores at 1733268302943 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733268302943Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733268302943Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733268302943Cleaning up temporary data from old regions at 1733268302951 (+8 ms)Running coprocessor post-open hooks at 1733268302954 (+3 ms)Region opened successfully at 1733268302954 2024-12-03T23:25:02,965 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 22ac440ef159085837f649a68330baac 3/3 column families, dataSize=590 B heapSize=2.08 KB 2024-12-03T23:25:02,966 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 22ac440ef159085837f649a68330baac/a, retrying num=0 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T23:25:03,257 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenViaHRegion 2024-12-03T23:25:03,258 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenViaHRegion Metrics about Tables on a single HBase RegionServer 2024-12-03T23:25:03,259 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testSequentialEditLogSeqNum 2024-12-03T23:25:03,259 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testSequentialEditLogSeqNum Metrics about Tables on a single HBase RegionServer 2024-12-03T23:25:03,260 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterRegionMovedWithMultiCF 2024-12-03T23:25:03,260 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterRegionMovedWithMultiCF Metrics about Tables on a single HBase RegionServer 2024-12-03T23:25:03,260 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRegionMadeOfBulkLoadedFilesOnly 2024-12-03T23:25:03,260 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testRegionMadeOfBulkLoadedFilesOnly Metrics about Tables on a single HBase RegionServer 2024-12-03T23:25:03,261 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterAbortingFlush 2024-12-03T23:25:03,261 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterAbortingFlush Metrics about Tables on a single HBase RegionServer 2024-12-03T23:25:03,967 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 22ac440ef159085837f649a68330baac/a, retrying num=1 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T23:25:04,179 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-03T23:25:04,968 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 22ac440ef159085837f649a68330baac/a, retrying num=2 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T23:25:05,968 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 22ac440ef159085837f649a68330baac/a, retrying num=3 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T23:25:06,969 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 22ac440ef159085837f649a68330baac/a, retrying num=4 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T23:25:07,970 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 22ac440ef159085837f649a68330baac/a, retrying num=5 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T23:25:08,970 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 22ac440ef159085837f649a68330baac/a, retrying num=6 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T23:25:09,971 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 22ac440ef159085837f649a68330baac/a, retrying num=7 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T23:25:10,972 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 22ac440ef159085837f649a68330baac/a, retrying num=8 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T23:25:11,973 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 22ac440ef159085837f649a68330baac/a, retrying num=9 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T23:25:11,974 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 22ac440ef159085837f649a68330baac: 2024-12-03T23:25:11,974 INFO [Time-limited test {}] wal.AbstractTestWALReplay(671): Expected simulated exception when flushing region, region: testReplayEditsAfterAbortingFlush,,1733268302794.22ac440ef159085837f649a68330baac. 2024-12-03T23:25:11,983 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-03T23:25:11,988 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 22ac440ef159085837f649a68330baac: 2024-12-03T23:25:11,988 INFO [Time-limited test {}] wal.AbstractTestWALReplay(691): Expected exception when flushing region because server is stopped,Aborting flush because server is aborted... 2024-12-03T23:25:11,988 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 22ac440ef159085837f649a68330baac, disabling compactions & flushes 2024-12-03T23:25:11,988 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterAbortingFlush,,1733268302794.22ac440ef159085837f649a68330baac. 2024-12-03T23:25:11,988 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterAbortingFlush,,1733268302794.22ac440ef159085837f649a68330baac. 2024-12-03T23:25:11,988 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterAbortingFlush,,1733268302794.22ac440ef159085837f649a68330baac. after waiting 0 ms 2024-12-03T23:25:11,988 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterAbortingFlush,,1733268302794.22ac440ef159085837f649a68330baac. 2024-12-03T23:25:11,989 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 1190 in region testReplayEditsAfterAbortingFlush,,1733268302794.22ac440ef159085837f649a68330baac. 2024-12-03T23:25:11,989 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsAfterAbortingFlush,,1733268302794.22ac440ef159085837f649a68330baac. 2024-12-03T23:25:11,989 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 22ac440ef159085837f649a68330baac: Waiting for close lock at 1733268311988Running coprocessor pre-close hooks at 1733268311988Disabling compacts and flushes for region at 1733268311988Disabling writes for close at 1733268311988Writing region close event to WAL at 1733268311989 (+1 ms)Running coprocessor post-close hooks at 1733268311989Closed at 1733268311989 2024-12-03T23:25:11,992 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39829 is added to blk_1073741908_1086 (size=2398) 2024-12-03T23:25:11,992 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41487 is added to blk_1073741908_1086 (size=2398) 2024-12-03T23:25:11,992 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45491 is added to blk_1073741908_1086 (size=2398) 2024-12-03T23:25:12,017 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:35563/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733268302792/wal.1733268302855, size=2.3 K (2398bytes) 2024-12-03T23:25:12,017 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:35563/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733268302792/wal.1733268302855 2024-12-03T23:25:12,017 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:35563/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733268302792/wal.1733268302855 after 0ms 2024-12-03T23:25:12,020 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:35563/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733268302792/wal.1733268302855: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-03T23:25:12,021 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:35563/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733268302792/wal.1733268302855 took 4ms 2024-12-03T23:25:12,023 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:35563/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733268302792/wal.1733268302855 so closing down 2024-12-03T23:25:12,023 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-03T23:25:12,024 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-wal.1733268302855.temp 2024-12-03T23:25:12,025 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:35563/hbase/data/default/testReplayEditsAfterAbortingFlush/22ac440ef159085837f649a68330baac/recovered.edits/0000000000000000004-wal.1733268302855.temp 2024-12-03T23:25:12,025 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-03T23:25:12,031 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45491 is added to blk_1073741909_1087 (size=1672) 2024-12-03T23:25:12,031 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41487 is added to blk_1073741909_1087 (size=1672) 2024-12-03T23:25:12,031 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39829 is added to blk_1073741909_1087 (size=1672) 2024-12-03T23:25:12,032 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:35563/hbase/data/default/testReplayEditsAfterAbortingFlush/22ac440ef159085837f649a68330baac/recovered.edits/0000000000000000004-wal.1733268302855.temp (wrote 20 edits, skipped 0 edits in 0 ms) 2024-12-03T23:25:12,033 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:35563/hbase/data/default/testReplayEditsAfterAbortingFlush/22ac440ef159085837f649a68330baac/recovered.edits/0000000000000000004-wal.1733268302855.temp to hdfs://localhost:35563/hbase/data/default/testReplayEditsAfterAbortingFlush/22ac440ef159085837f649a68330baac/recovered.edits/0000000000000000026 2024-12-03T23:25:12,033 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 23 edits across 1 Regions in 12 ms; skipped=3; WAL=hdfs://localhost:35563/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733268302792/wal.1733268302855, size=2.3 K, length=2398, corrupted=false, cancelled=false 2024-12-03T23:25:12,034 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:35563/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733268302792/wal.1733268302855, journal: Splitting hdfs://localhost:35563/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733268302792/wal.1733268302855, size=2.3 K (2398bytes) at 1733268312017Finishing writing output for hdfs://localhost:35563/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733268302792/wal.1733268302855 so closing down at 1733268312023 (+6 ms)Creating recovered edits writer path=hdfs://localhost:35563/hbase/data/default/testReplayEditsAfterAbortingFlush/22ac440ef159085837f649a68330baac/recovered.edits/0000000000000000004-wal.1733268302855.temp at 1733268312025 (+2 ms)3 split writer threads finished at 1733268312025Closed recovered edits writer path=hdfs://localhost:35563/hbase/data/default/testReplayEditsAfterAbortingFlush/22ac440ef159085837f649a68330baac/recovered.edits/0000000000000000004-wal.1733268302855.temp (wrote 20 edits, skipped 0 edits in 0 ms) at 1733268312032 (+7 ms)Rename recovered edits hdfs://localhost:35563/hbase/data/default/testReplayEditsAfterAbortingFlush/22ac440ef159085837f649a68330baac/recovered.edits/0000000000000000004-wal.1733268302855.temp to hdfs://localhost:35563/hbase/data/default/testReplayEditsAfterAbortingFlush/22ac440ef159085837f649a68330baac/recovered.edits/0000000000000000026 at 1733268312033 (+1 ms)Processed 23 edits across 1 Regions in 12 ms; skipped=3; WAL=hdfs://localhost:35563/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733268302792/wal.1733268302855, size=2.3 K, length=2398, corrupted=false, cancelled=false at 1733268312033 2024-12-03T23:25:12,035 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:35563/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733268302792/wal.1733268302855 to hdfs://localhost:35563/hbase/oldWALs/wal.1733268302855 2024-12-03T23:25:12,036 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:35563/hbase/data/default/testReplayEditsAfterAbortingFlush/22ac440ef159085837f649a68330baac/recovered.edits/0000000000000000026 2024-12-03T23:25:12,036 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-03T23:25:12,037 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:35563/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733268302792, archiveDir=hdfs://localhost:35563/hbase/oldWALs, maxLogs=32 2024-12-03T23:25:12,053 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733268302792/wal.1733268312038, exclude list is [], retry=0 2024-12-03T23:25:12,056 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41487,DS-459d168f-61f7-403b-9139-91eca369c54e,DISK] 2024-12-03T23:25:12,056 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39829,DS-e6270720-ef81-458e-b5d6-959d16c4c378,DISK] 2024-12-03T23:25:12,057 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45491,DS-d55e5535-c0ce-4dc2-a45c-8e70eb010e56,DISK] 2024-12-03T23:25:12,058 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733268302792/wal.1733268312038 2024-12-03T23:25:12,058 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36361:36361),(127.0.0.1/127.0.0.1:38065:38065),(127.0.0.1/127.0.0.1:34451:34451)] 2024-12-03T23:25:12,059 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 22ac440ef159085837f649a68330baac, NAME => 'testReplayEditsAfterAbortingFlush,,1733268302794.22ac440ef159085837f649a68330baac.', STARTKEY => '', ENDKEY => ''} 2024-12-03T23:25:12,059 DEBUG [Time-limited test {}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterAbortingFlush 22ac440ef159085837f649a68330baac 2024-12-03T23:25:12,059 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterAbortingFlush,,1733268302794.22ac440ef159085837f649a68330baac.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T23:25:12,060 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 22ac440ef159085837f649a68330baac 2024-12-03T23:25:12,060 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 22ac440ef159085837f649a68330baac 2024-12-03T23:25:12,061 INFO [StoreOpener-22ac440ef159085837f649a68330baac-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 22ac440ef159085837f649a68330baac 2024-12-03T23:25:12,062 INFO [StoreOpener-22ac440ef159085837f649a68330baac-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 22ac440ef159085837f649a68330baac columnFamilyName a 2024-12-03T23:25:12,062 DEBUG [StoreOpener-22ac440ef159085837f649a68330baac-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T23:25:12,063 INFO [StoreOpener-22ac440ef159085837f649a68330baac-1 {}] regionserver.HStore(327): Store=22ac440ef159085837f649a68330baac/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T23:25:12,063 INFO [StoreOpener-22ac440ef159085837f649a68330baac-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 22ac440ef159085837f649a68330baac 2024-12-03T23:25:12,064 INFO [StoreOpener-22ac440ef159085837f649a68330baac-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 22ac440ef159085837f649a68330baac columnFamilyName b 2024-12-03T23:25:12,064 DEBUG [StoreOpener-22ac440ef159085837f649a68330baac-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T23:25:12,064 INFO [StoreOpener-22ac440ef159085837f649a68330baac-1 {}] regionserver.HStore(327): Store=22ac440ef159085837f649a68330baac/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T23:25:12,065 INFO [StoreOpener-22ac440ef159085837f649a68330baac-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 22ac440ef159085837f649a68330baac 2024-12-03T23:25:12,065 INFO [StoreOpener-22ac440ef159085837f649a68330baac-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 22ac440ef159085837f649a68330baac columnFamilyName c 2024-12-03T23:25:12,065 DEBUG [StoreOpener-22ac440ef159085837f649a68330baac-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T23:25:12,066 INFO [StoreOpener-22ac440ef159085837f649a68330baac-1 {}] regionserver.HStore(327): Store=22ac440ef159085837f649a68330baac/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T23:25:12,066 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 22ac440ef159085837f649a68330baac 2024-12-03T23:25:12,067 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35563/hbase/data/default/testReplayEditsAfterAbortingFlush/22ac440ef159085837f649a68330baac 2024-12-03T23:25:12,068 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:35563/hbase/data/default/testReplayEditsAfterAbortingFlush/22ac440ef159085837f649a68330baac 2024-12-03T23:25:12,069 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:35563/hbase/data/default/testReplayEditsAfterAbortingFlush/22ac440ef159085837f649a68330baac/recovered.edits/0000000000000000026 2024-12-03T23:25:12,071 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:35563/hbase/data/default/testReplayEditsAfterAbortingFlush/22ac440ef159085837f649a68330baac/recovered.edits/0000000000000000026: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-03T23:25:12,073 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 20, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=26, path=hdfs://localhost:35563/hbase/data/default/testReplayEditsAfterAbortingFlush/22ac440ef159085837f649a68330baac/recovered.edits/0000000000000000026 2024-12-03T23:25:12,074 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 22ac440ef159085837f649a68330baac 3/3 column families, dataSize=1.16 KB heapSize=3.41 KB 2024-12-03T23:25:12,094 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35563/hbase/data/default/testReplayEditsAfterAbortingFlush/22ac440ef159085837f649a68330baac/.tmp/a/89ae870e8c3540f7ae67d99f2d63c841 is 64, key is testReplayEditsAfterAbortingFlush12/a:q/1733268311978/Put/seqid=0 2024-12-03T23:25:12,101 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41487 is added to blk_1073741911_1089 (size=5523) 2024-12-03T23:25:12,101 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39829 is added to blk_1073741911_1089 (size=5523) 2024-12-03T23:25:12,101 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45491 is added to blk_1073741911_1089 (size=5523) 2024-12-03T23:25:12,102 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=416 B at sequenceid=26 (bloomFilter=true), to=hdfs://localhost:35563/hbase/data/default/testReplayEditsAfterAbortingFlush/22ac440ef159085837f649a68330baac/.tmp/a/89ae870e8c3540f7ae67d99f2d63c841 2024-12-03T23:25:12,146 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35563/hbase/data/default/testReplayEditsAfterAbortingFlush/22ac440ef159085837f649a68330baac/.tmp/b/4b3130c35c7c4818aabf1e048eefbd36 is 64, key is testReplayEditsAfterAbortingFlush10/b:q/1733268311974/Put/seqid=0 2024-12-03T23:25:12,153 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45491 is added to blk_1073741912_1090 (size=5524) 2024-12-03T23:25:12,153 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41487 is added to blk_1073741912_1090 (size=5524) 2024-12-03T23:25:12,153 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39829 is added to blk_1073741912_1090 (size=5524) 2024-12-03T23:25:12,154 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=417 B at sequenceid=26 (bloomFilter=true), to=hdfs://localhost:35563/hbase/data/default/testReplayEditsAfterAbortingFlush/22ac440ef159085837f649a68330baac/.tmp/b/4b3130c35c7c4818aabf1e048eefbd36 2024-12-03T23:25:12,173 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35563/hbase/data/default/testReplayEditsAfterAbortingFlush/22ac440ef159085837f649a68330baac/.tmp/c/ea6fb3af18824e6b8d0a84001c168a5e is 64, key is testReplayEditsAfterAbortingFlush11/c:q/1733268311976/Put/seqid=0 2024-12-03T23:25:12,183 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45491 is added to blk_1073741913_1091 (size=5457) 2024-12-03T23:25:12,183 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39829 is added to blk_1073741913_1091 (size=5457) 2024-12-03T23:25:12,183 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41487 is added to blk_1073741913_1091 (size=5457) 2024-12-03T23:25:12,184 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=357 B at sequenceid=26 (bloomFilter=true), to=hdfs://localhost:35563/hbase/data/default/testReplayEditsAfterAbortingFlush/22ac440ef159085837f649a68330baac/.tmp/c/ea6fb3af18824e6b8d0a84001c168a5e 2024-12-03T23:25:12,190 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35563/hbase/data/default/testReplayEditsAfterAbortingFlush/22ac440ef159085837f649a68330baac/.tmp/a/89ae870e8c3540f7ae67d99f2d63c841 as hdfs://localhost:35563/hbase/data/default/testReplayEditsAfterAbortingFlush/22ac440ef159085837f649a68330baac/a/89ae870e8c3540f7ae67d99f2d63c841 2024-12-03T23:25:12,196 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35563/hbase/data/default/testReplayEditsAfterAbortingFlush/22ac440ef159085837f649a68330baac/a/89ae870e8c3540f7ae67d99f2d63c841, entries=7, sequenceid=26, filesize=5.4 K 2024-12-03T23:25:12,197 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35563/hbase/data/default/testReplayEditsAfterAbortingFlush/22ac440ef159085837f649a68330baac/.tmp/b/4b3130c35c7c4818aabf1e048eefbd36 as hdfs://localhost:35563/hbase/data/default/testReplayEditsAfterAbortingFlush/22ac440ef159085837f649a68330baac/b/4b3130c35c7c4818aabf1e048eefbd36 2024-12-03T23:25:12,202 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35563/hbase/data/default/testReplayEditsAfterAbortingFlush/22ac440ef159085837f649a68330baac/b/4b3130c35c7c4818aabf1e048eefbd36, entries=7, sequenceid=26, filesize=5.4 K 2024-12-03T23:25:12,203 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35563/hbase/data/default/testReplayEditsAfterAbortingFlush/22ac440ef159085837f649a68330baac/.tmp/c/ea6fb3af18824e6b8d0a84001c168a5e as hdfs://localhost:35563/hbase/data/default/testReplayEditsAfterAbortingFlush/22ac440ef159085837f649a68330baac/c/ea6fb3af18824e6b8d0a84001c168a5e 2024-12-03T23:25:12,209 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35563/hbase/data/default/testReplayEditsAfterAbortingFlush/22ac440ef159085837f649a68330baac/c/ea6fb3af18824e6b8d0a84001c168a5e, entries=6, sequenceid=26, filesize=5.3 K 2024-12-03T23:25:12,209 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~1.16 KB/1190, heapSize ~3.36 KB/3440, currentSize=0 B/0 for 22ac440ef159085837f649a68330baac in 136ms, sequenceid=26, compaction requested=false; wal=null 2024-12-03T23:25:12,210 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:35563/hbase/data/default/testReplayEditsAfterAbortingFlush/22ac440ef159085837f649a68330baac/recovered.edits/0000000000000000026 2024-12-03T23:25:12,211 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 22ac440ef159085837f649a68330baac 2024-12-03T23:25:12,211 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 22ac440ef159085837f649a68330baac 2024-12-03T23:25:12,212 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterAbortingFlush descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-03T23:25:12,213 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 22ac440ef159085837f649a68330baac 2024-12-03T23:25:12,215 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35563/hbase/data/default/testReplayEditsAfterAbortingFlush/22ac440ef159085837f649a68330baac/recovered.edits/26.seqid, newMaxSeqId=26, maxSeqId=1 2024-12-03T23:25:12,216 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 22ac440ef159085837f649a68330baac; next sequenceid=27; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65008755, jitterRate=-0.03129406273365021}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-03T23:25:12,216 DEBUG [Time-limited test {}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 22ac440ef159085837f649a68330baac 2024-12-03T23:25:12,217 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 22ac440ef159085837f649a68330baac: Running coprocessor pre-open hook at 1733268312060Writing region info on filesystem at 1733268312060Initializing all the Stores at 1733268312061 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733268312061Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733268312061Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733268312061Obtaining lock to block concurrent updates at 1733268312074 (+13 ms)Preparing flush snapshotting stores in 22ac440ef159085837f649a68330baac at 1733268312074Finished memstore snapshotting testReplayEditsAfterAbortingFlush,,1733268302794.22ac440ef159085837f649a68330baac., syncing WAL and waiting on mvcc, flushsize=dataSize=1190, getHeapSize=3440, getOffHeapSize=0, getCellsCount=20 at 1733268312074Flushing stores of testReplayEditsAfterAbortingFlush,,1733268302794.22ac440ef159085837f649a68330baac. at 1733268312074Flushing 22ac440ef159085837f649a68330baac/a: creating writer at 1733268312074Flushing 22ac440ef159085837f649a68330baac/a: appending metadata at 1733268312093 (+19 ms)Flushing 22ac440ef159085837f649a68330baac/a: closing flushed file at 1733268312093Flushing 22ac440ef159085837f649a68330baac/b: creating writer at 1733268312129 (+36 ms)Flushing 22ac440ef159085837f649a68330baac/b: appending metadata at 1733268312145 (+16 ms)Flushing 22ac440ef159085837f649a68330baac/b: closing flushed file at 1733268312145Flushing 22ac440ef159085837f649a68330baac/c: creating writer at 1733268312159 (+14 ms)Flushing 22ac440ef159085837f649a68330baac/c: appending metadata at 1733268312173 (+14 ms)Flushing 22ac440ef159085837f649a68330baac/c: closing flushed file at 1733268312173Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7468f5a2: reopening flushed file at 1733268312189 (+16 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4b52e8db: reopening flushed file at 1733268312197 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7952c5db: reopening flushed file at 1733268312203 (+6 ms)Finished flush of dataSize ~1.16 KB/1190, heapSize ~3.36 KB/3440, currentSize=0 B/0 for 22ac440ef159085837f649a68330baac in 136ms, sequenceid=26, compaction requested=false; wal=null at 1733268312209 (+6 ms)Cleaning up temporary data from old regions at 1733268312211 (+2 ms)Running coprocessor post-open hooks at 1733268312216 (+5 ms)Region opened successfully at 1733268312217 (+1 ms) 2024-12-03T23:25:12,241 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayCompressed#testReplayEditsAfterAbortingFlush Thread=419 (was 419), OpenFileDescriptor=1180 (was 1124) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=210 (was 222), ProcessCount=11 (was 11), AvailableMemoryMB=7812 (was 7918) 2024-12-03T23:25:12,241 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1180 is superior to 1024 2024-12-03T23:25:12,255 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayCompressed#testDatalossWhenInputError Thread=419, OpenFileDescriptor=1180, MaxFileDescriptor=1048576, SystemLoadAverage=210, ProcessCount=11, AvailableMemoryMB=7811 2024-12-03T23:25:12,255 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1180 is superior to 1024 2024-12-03T23:25:12,278 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-03T23:25:12,280 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-03T23:25:12,281 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-03T23:25:12,283 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-19089890, suffix=, logDir=hdfs://localhost:35563/hbase/WALs/hregion-19089890, archiveDir=hdfs://localhost:35563/hbase/oldWALs, maxLogs=32 2024-12-03T23:25:12,296 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-19089890/hregion-19089890.1733268312283, exclude list is [], retry=0 2024-12-03T23:25:12,299 DEBUG [AsyncFSWAL-24-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39829,DS-e6270720-ef81-458e-b5d6-959d16c4c378,DISK] 2024-12-03T23:25:12,299 DEBUG [AsyncFSWAL-24-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45491,DS-d55e5535-c0ce-4dc2-a45c-8e70eb010e56,DISK] 2024-12-03T23:25:12,299 DEBUG [AsyncFSWAL-24-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41487,DS-459d168f-61f7-403b-9139-91eca369c54e,DISK] 2024-12-03T23:25:12,301 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-19089890/hregion-19089890.1733268312283 2024-12-03T23:25:12,301 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:38065:38065),(127.0.0.1/127.0.0.1:34451:34451),(127.0.0.1/127.0.0.1:36361:36361)] 2024-12-03T23:25:12,301 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 97eecf578d8a929685b4d07f76362d71, NAME => 'testDatalossWhenInputError,,1733268312279.97eecf578d8a929685b4d07f76362d71.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testDatalossWhenInputError', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35563/hbase 2024-12-03T23:25:12,310 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45491 is added to blk_1073741915_1093 (size=61) 2024-12-03T23:25:12,310 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41487 is added to blk_1073741915_1093 (size=61) 2024-12-03T23:25:12,310 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39829 is added to blk_1073741915_1093 (size=61) 2024-12-03T23:25:12,311 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1733268312279.97eecf578d8a929685b4d07f76362d71.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T23:25:12,312 INFO [StoreOpener-97eecf578d8a929685b4d07f76362d71-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 97eecf578d8a929685b4d07f76362d71 2024-12-03T23:25:12,314 INFO [StoreOpener-97eecf578d8a929685b4d07f76362d71-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 97eecf578d8a929685b4d07f76362d71 columnFamilyName a 2024-12-03T23:25:12,314 DEBUG [StoreOpener-97eecf578d8a929685b4d07f76362d71-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T23:25:12,315 INFO [StoreOpener-97eecf578d8a929685b4d07f76362d71-1 {}] regionserver.HStore(327): Store=97eecf578d8a929685b4d07f76362d71/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T23:25:12,315 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 97eecf578d8a929685b4d07f76362d71 2024-12-03T23:25:12,315 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35563/hbase/data/default/testDatalossWhenInputError/97eecf578d8a929685b4d07f76362d71 2024-12-03T23:25:12,316 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35563/hbase/data/default/testDatalossWhenInputError/97eecf578d8a929685b4d07f76362d71 2024-12-03T23:25:12,316 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 97eecf578d8a929685b4d07f76362d71 2024-12-03T23:25:12,316 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 97eecf578d8a929685b4d07f76362d71 2024-12-03T23:25:12,318 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 97eecf578d8a929685b4d07f76362d71 2024-12-03T23:25:12,320 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35563/hbase/data/default/testDatalossWhenInputError/97eecf578d8a929685b4d07f76362d71/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T23:25:12,320 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 97eecf578d8a929685b4d07f76362d71; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59593431, jitterRate=-0.11198867857456207}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-03T23:25:12,321 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 97eecf578d8a929685b4d07f76362d71: Writing region info on filesystem at 1733268312311Initializing all the Stores at 1733268312312 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733268312312Cleaning up temporary data from old regions at 1733268312316 (+4 ms)Region opened successfully at 1733268312321 (+5 ms) 2024-12-03T23:25:12,321 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 97eecf578d8a929685b4d07f76362d71, disabling compactions & flushes 2024-12-03T23:25:12,321 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testDatalossWhenInputError,,1733268312279.97eecf578d8a929685b4d07f76362d71. 2024-12-03T23:25:12,321 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testDatalossWhenInputError,,1733268312279.97eecf578d8a929685b4d07f76362d71. 2024-12-03T23:25:12,321 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testDatalossWhenInputError,,1733268312279.97eecf578d8a929685b4d07f76362d71. after waiting 0 ms 2024-12-03T23:25:12,321 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testDatalossWhenInputError,,1733268312279.97eecf578d8a929685b4d07f76362d71. 2024-12-03T23:25:12,322 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testDatalossWhenInputError,,1733268312279.97eecf578d8a929685b4d07f76362d71. 2024-12-03T23:25:12,322 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 97eecf578d8a929685b4d07f76362d71: Waiting for close lock at 1733268312321Disabling compacts and flushes for region at 1733268312321Disabling writes for close at 1733268312321Writing region close event to WAL at 1733268312322 (+1 ms)Closed at 1733268312322 2024-12-03T23:25:12,324 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39829 is added to blk_1073741914_1092 (size=93) 2024-12-03T23:25:12,325 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45491 is added to blk_1073741914_1092 (size=93) 2024-12-03T23:25:12,325 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41487 is added to blk_1073741914_1092 (size=93) 2024-12-03T23:25:12,327 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-03T23:25:12,327 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-19089890:(num 1733268312283) 2024-12-03T23:25:12,327 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-03T23:25:12,329 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:35563/hbase/WALs/testdatalosswheninputerror-manual,16010,1733268312277, archiveDir=hdfs://localhost:35563/hbase/oldWALs, maxLogs=32 2024-12-03T23:25:12,342 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testdatalosswheninputerror-manual,16010,1733268312277/wal.1733268312329, exclude list is [], retry=0 2024-12-03T23:25:12,345 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41487,DS-459d168f-61f7-403b-9139-91eca369c54e,DISK] 2024-12-03T23:25:12,345 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39829,DS-e6270720-ef81-458e-b5d6-959d16c4c378,DISK] 2024-12-03T23:25:12,345 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45491,DS-d55e5535-c0ce-4dc2-a45c-8e70eb010e56,DISK] 2024-12-03T23:25:12,347 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testdatalosswheninputerror-manual,16010,1733268312277/wal.1733268312329 2024-12-03T23:25:12,347 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36361:36361),(127.0.0.1/127.0.0.1:38065:38065),(127.0.0.1/127.0.0.1:34451:34451)] 2024-12-03T23:25:12,347 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 97eecf578d8a929685b4d07f76362d71, NAME => 'testDatalossWhenInputError,,1733268312279.97eecf578d8a929685b4d07f76362d71.', STARTKEY => '', ENDKEY => ''} 2024-12-03T23:25:12,347 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1733268312279.97eecf578d8a929685b4d07f76362d71.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T23:25:12,347 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 97eecf578d8a929685b4d07f76362d71 2024-12-03T23:25:12,347 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 97eecf578d8a929685b4d07f76362d71 2024-12-03T23:25:12,349 INFO [StoreOpener-97eecf578d8a929685b4d07f76362d71-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 97eecf578d8a929685b4d07f76362d71 2024-12-03T23:25:12,349 INFO [StoreOpener-97eecf578d8a929685b4d07f76362d71-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 97eecf578d8a929685b4d07f76362d71 columnFamilyName a 2024-12-03T23:25:12,349 DEBUG [StoreOpener-97eecf578d8a929685b4d07f76362d71-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T23:25:12,350 INFO [StoreOpener-97eecf578d8a929685b4d07f76362d71-1 {}] regionserver.HStore(327): Store=97eecf578d8a929685b4d07f76362d71/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T23:25:12,350 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 97eecf578d8a929685b4d07f76362d71 2024-12-03T23:25:12,351 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35563/hbase/data/default/testDatalossWhenInputError/97eecf578d8a929685b4d07f76362d71 2024-12-03T23:25:12,352 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35563/hbase/data/default/testDatalossWhenInputError/97eecf578d8a929685b4d07f76362d71 2024-12-03T23:25:12,352 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 97eecf578d8a929685b4d07f76362d71 2024-12-03T23:25:12,352 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 97eecf578d8a929685b4d07f76362d71 2024-12-03T23:25:12,354 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 97eecf578d8a929685b4d07f76362d71 2024-12-03T23:25:12,354 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 97eecf578d8a929685b4d07f76362d71; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62757687, jitterRate=-0.06483758985996246}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-03T23:25:12,355 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 97eecf578d8a929685b4d07f76362d71: Writing region info on filesystem at 1733268312347Initializing all the Stores at 1733268312348 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733268312348Cleaning up temporary data from old regions at 1733268312352 (+4 ms)Region opened successfully at 1733268312355 (+3 ms) 2024-12-03T23:25:12,364 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 97eecf578d8a929685b4d07f76362d71, disabling compactions & flushes 2024-12-03T23:25:12,364 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testDatalossWhenInputError,,1733268312279.97eecf578d8a929685b4d07f76362d71. 2024-12-03T23:25:12,364 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testDatalossWhenInputError,,1733268312279.97eecf578d8a929685b4d07f76362d71. 2024-12-03T23:25:12,365 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testDatalossWhenInputError,,1733268312279.97eecf578d8a929685b4d07f76362d71. after waiting 0 ms 2024-12-03T23:25:12,365 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testDatalossWhenInputError,,1733268312279.97eecf578d8a929685b4d07f76362d71. 2024-12-03T23:25:12,365 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 750 in region testDatalossWhenInputError,,1733268312279.97eecf578d8a929685b4d07f76362d71. 2024-12-03T23:25:12,365 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testDatalossWhenInputError,,1733268312279.97eecf578d8a929685b4d07f76362d71. 2024-12-03T23:25:12,365 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 97eecf578d8a929685b4d07f76362d71: Waiting for close lock at 1733268312364Disabling compacts and flushes for region at 1733268312364Disabling writes for close at 1733268312365 (+1 ms)Writing region close event to WAL at 1733268312365Closed at 1733268312365 2024-12-03T23:25:12,368 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45491 is added to blk_1073741916_1094 (size=838) 2024-12-03T23:25:12,368 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41487 is added to blk_1073741916_1094 (size=838) 2024-12-03T23:25:12,368 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39829 is added to blk_1073741916_1094 (size=838) 2024-12-03T23:25:12,385 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:35563/hbase/WALs/testdatalosswheninputerror-manual,16010,1733268312277/wal.1733268312329, size=838 (838bytes) 2024-12-03T23:25:12,385 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:35563/hbase/WALs/testdatalosswheninputerror-manual,16010,1733268312277/wal.1733268312329 2024-12-03T23:25:12,385 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:35563/hbase/WALs/testdatalosswheninputerror-manual,16010,1733268312277/wal.1733268312329 after 0ms 2024-12-03T23:25:12,387 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:35563/hbase/WALs/testdatalosswheninputerror-manual,16010,1733268312277/wal.1733268312329: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-03T23:25:12,388 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:35563/hbase/WALs/testdatalosswheninputerror-manual,16010,1733268312277/wal.1733268312329 took 4ms 2024-12-03T23:25:12,389 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:35563/hbase/WALs/testdatalosswheninputerror-manual,16010,1733268312277/wal.1733268312329 so closing down 2024-12-03T23:25:12,389 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-03T23:25:12,391 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1733268312329.temp 2024-12-03T23:25:12,392 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:35563/hbase/data/default/testDatalossWhenInputError/97eecf578d8a929685b4d07f76362d71/recovered.edits/0000000000000000003-wal.1733268312329.temp 2024-12-03T23:25:12,392 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-03T23:25:12,400 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39829 is added to blk_1073741917_1095 (size=838) 2024-12-03T23:25:12,400 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41487 is added to blk_1073741917_1095 (size=838) 2024-12-03T23:25:12,401 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45491 is added to blk_1073741917_1095 (size=838) 2024-12-03T23:25:12,401 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:35563/hbase/data/default/testDatalossWhenInputError/97eecf578d8a929685b4d07f76362d71/recovered.edits/0000000000000000003-wal.1733268312329.temp (wrote 10 edits, skipped 0 edits in 0 ms) 2024-12-03T23:25:12,402 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:35563/hbase/data/default/testDatalossWhenInputError/97eecf578d8a929685b4d07f76362d71/recovered.edits/0000000000000000003-wal.1733268312329.temp to hdfs://localhost:35563/hbase/data/default/testDatalossWhenInputError/97eecf578d8a929685b4d07f76362d71/recovered.edits/0000000000000000012 2024-12-03T23:25:12,403 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 10 edits across 1 Regions in 15 ms; skipped=0; WAL=hdfs://localhost:35563/hbase/WALs/testdatalosswheninputerror-manual,16010,1733268312277/wal.1733268312329, size=838, length=838, corrupted=false, cancelled=false 2024-12-03T23:25:12,403 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:35563/hbase/WALs/testdatalosswheninputerror-manual,16010,1733268312277/wal.1733268312329, journal: Splitting hdfs://localhost:35563/hbase/WALs/testdatalosswheninputerror-manual,16010,1733268312277/wal.1733268312329, size=838 (838bytes) at 1733268312385Finishing writing output for hdfs://localhost:35563/hbase/WALs/testdatalosswheninputerror-manual,16010,1733268312277/wal.1733268312329 so closing down at 1733268312389 (+4 ms)Creating recovered edits writer path=hdfs://localhost:35563/hbase/data/default/testDatalossWhenInputError/97eecf578d8a929685b4d07f76362d71/recovered.edits/0000000000000000003-wal.1733268312329.temp at 1733268312392 (+3 ms)3 split writer threads finished at 1733268312392Closed recovered edits writer path=hdfs://localhost:35563/hbase/data/default/testDatalossWhenInputError/97eecf578d8a929685b4d07f76362d71/recovered.edits/0000000000000000003-wal.1733268312329.temp (wrote 10 edits, skipped 0 edits in 0 ms) at 1733268312401 (+9 ms)Rename recovered edits hdfs://localhost:35563/hbase/data/default/testDatalossWhenInputError/97eecf578d8a929685b4d07f76362d71/recovered.edits/0000000000000000003-wal.1733268312329.temp to hdfs://localhost:35563/hbase/data/default/testDatalossWhenInputError/97eecf578d8a929685b4d07f76362d71/recovered.edits/0000000000000000012 at 1733268312402 (+1 ms)Processed 10 edits across 1 Regions in 15 ms; skipped=0; WAL=hdfs://localhost:35563/hbase/WALs/testdatalosswheninputerror-manual,16010,1733268312277/wal.1733268312329, size=838, length=838, corrupted=false, cancelled=false at 1733268312403 (+1 ms) 2024-12-03T23:25:12,404 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:35563/hbase/WALs/testdatalosswheninputerror-manual,16010,1733268312277/wal.1733268312329 to hdfs://localhost:35563/hbase/oldWALs/wal.1733268312329 2024-12-03T23:25:12,405 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:35563/hbase/data/default/testDatalossWhenInputError/97eecf578d8a929685b4d07f76362d71/recovered.edits/0000000000000000012 2024-12-03T23:25:12,408 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:35563/hbase/data/default/testDatalossWhenInputError/97eecf578d8a929685b4d07f76362d71/recovered.edits/0000000000000000012: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-03T23:25:12,753 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-03T23:25:12,755 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:35563/hbase/WALs/testdatalosswheninputerror-manual,16010,1733268312277, archiveDir=hdfs://localhost:35563/hbase/oldWALs, maxLogs=32 2024-12-03T23:25:12,769 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testdatalosswheninputerror-manual,16010,1733268312277/wal.1733268312756, exclude list is [], retry=0 2024-12-03T23:25:12,771 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39829,DS-e6270720-ef81-458e-b5d6-959d16c4c378,DISK] 2024-12-03T23:25:12,772 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41487,DS-459d168f-61f7-403b-9139-91eca369c54e,DISK] 2024-12-03T23:25:12,772 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45491,DS-d55e5535-c0ce-4dc2-a45c-8e70eb010e56,DISK] 2024-12-03T23:25:12,773 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testdatalosswheninputerror-manual,16010,1733268312277/wal.1733268312756 2024-12-03T23:25:12,774 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:38065:38065),(127.0.0.1/127.0.0.1:36361:36361),(127.0.0.1/127.0.0.1:34451:34451)] 2024-12-03T23:25:12,774 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 97eecf578d8a929685b4d07f76362d71, NAME => 'testDatalossWhenInputError,,1733268312279.97eecf578d8a929685b4d07f76362d71.', STARTKEY => '', ENDKEY => ''} 2024-12-03T23:25:12,774 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1733268312279.97eecf578d8a929685b4d07f76362d71.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T23:25:12,774 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 97eecf578d8a929685b4d07f76362d71 2024-12-03T23:25:12,774 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 97eecf578d8a929685b4d07f76362d71 2024-12-03T23:25:12,777 INFO [StoreOpener-97eecf578d8a929685b4d07f76362d71-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 97eecf578d8a929685b4d07f76362d71 2024-12-03T23:25:12,778 INFO [StoreOpener-97eecf578d8a929685b4d07f76362d71-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 97eecf578d8a929685b4d07f76362d71 columnFamilyName a 2024-12-03T23:25:12,778 DEBUG [StoreOpener-97eecf578d8a929685b4d07f76362d71-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T23:25:12,780 INFO [StoreOpener-97eecf578d8a929685b4d07f76362d71-1 {}] regionserver.HStore(327): Store=97eecf578d8a929685b4d07f76362d71/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T23:25:12,780 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 97eecf578d8a929685b4d07f76362d71 2024-12-03T23:25:12,780 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35563/hbase/data/default/testDatalossWhenInputError/97eecf578d8a929685b4d07f76362d71 2024-12-03T23:25:12,782 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:35563/hbase/data/default/testDatalossWhenInputError/97eecf578d8a929685b4d07f76362d71 2024-12-03T23:25:12,782 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:35563/hbase/data/default/testDatalossWhenInputError/97eecf578d8a929685b4d07f76362d71/recovered.edits/0000000000000000012 2024-12-03T23:25:12,784 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:35563/hbase/data/default/testDatalossWhenInputError/97eecf578d8a929685b4d07f76362d71/recovered.edits/0000000000000000012: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-03T23:25:12,785 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 10, skipped 0, firstSequenceIdInLog=3, maxSequenceIdInLog=12, path=hdfs://localhost:35563/hbase/data/default/testDatalossWhenInputError/97eecf578d8a929685b4d07f76362d71/recovered.edits/0000000000000000012 2024-12-03T23:25:12,785 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 97eecf578d8a929685b4d07f76362d71 1/1 column families, dataSize=750 B heapSize=1.73 KB 2024-12-03T23:25:12,804 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35563/hbase/data/default/testDatalossWhenInputError/97eecf578d8a929685b4d07f76362d71/.tmp/a/b571caecfed44af3b7cd36c419698ab8 is 79, key is testDatalossWhenInputError/a:x0/1733268312355/Put/seqid=0 2024-12-03T23:25:12,810 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45491 is added to blk_1073741919_1097 (size=5808) 2024-12-03T23:25:12,810 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41487 is added to blk_1073741919_1097 (size=5808) 2024-12-03T23:25:12,811 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39829 is added to blk_1073741919_1097 (size=5808) 2024-12-03T23:25:12,811 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=750 B at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:35563/hbase/data/default/testDatalossWhenInputError/97eecf578d8a929685b4d07f76362d71/.tmp/a/b571caecfed44af3b7cd36c419698ab8 2024-12-03T23:25:12,819 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35563/hbase/data/default/testDatalossWhenInputError/97eecf578d8a929685b4d07f76362d71/.tmp/a/b571caecfed44af3b7cd36c419698ab8 as hdfs://localhost:35563/hbase/data/default/testDatalossWhenInputError/97eecf578d8a929685b4d07f76362d71/a/b571caecfed44af3b7cd36c419698ab8 2024-12-03T23:25:12,827 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35563/hbase/data/default/testDatalossWhenInputError/97eecf578d8a929685b4d07f76362d71/a/b571caecfed44af3b7cd36c419698ab8, entries=10, sequenceid=12, filesize=5.7 K 2024-12-03T23:25:12,827 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~750 B/750, heapSize ~1.72 KB/1760, currentSize=0 B/0 for 97eecf578d8a929685b4d07f76362d71 in 42ms, sequenceid=12, compaction requested=false; wal=null 2024-12-03T23:25:12,828 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:35563/hbase/data/default/testDatalossWhenInputError/97eecf578d8a929685b4d07f76362d71/recovered.edits/0000000000000000012 2024-12-03T23:25:12,829 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 97eecf578d8a929685b4d07f76362d71 2024-12-03T23:25:12,829 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 97eecf578d8a929685b4d07f76362d71 2024-12-03T23:25:12,831 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 97eecf578d8a929685b4d07f76362d71 2024-12-03T23:25:12,833 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35563/hbase/data/default/testDatalossWhenInputError/97eecf578d8a929685b4d07f76362d71/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=1 2024-12-03T23:25:12,834 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 97eecf578d8a929685b4d07f76362d71; next sequenceid=13; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73927506, jitterRate=0.10160568356513977}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-03T23:25:12,835 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 97eecf578d8a929685b4d07f76362d71: Writing region info on filesystem at 1733268312774Initializing all the Stores at 1733268312776 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733268312776Obtaining lock to block concurrent updates at 1733268312785 (+9 ms)Preparing flush snapshotting stores in 97eecf578d8a929685b4d07f76362d71 at 1733268312785Finished memstore snapshotting testDatalossWhenInputError,,1733268312279.97eecf578d8a929685b4d07f76362d71., syncing WAL and waiting on mvcc, flushsize=dataSize=750, getHeapSize=1760, getOffHeapSize=0, getCellsCount=10 at 1733268312785Flushing stores of testDatalossWhenInputError,,1733268312279.97eecf578d8a929685b4d07f76362d71. at 1733268312785Flushing 97eecf578d8a929685b4d07f76362d71/a: creating writer at 1733268312785Flushing 97eecf578d8a929685b4d07f76362d71/a: appending metadata at 1733268312803 (+18 ms)Flushing 97eecf578d8a929685b4d07f76362d71/a: closing flushed file at 1733268312804 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2d5719c2: reopening flushed file at 1733268312818 (+14 ms)Finished flush of dataSize ~750 B/750, heapSize ~1.72 KB/1760, currentSize=0 B/0 for 97eecf578d8a929685b4d07f76362d71 in 42ms, sequenceid=12, compaction requested=false; wal=null at 1733268312827 (+9 ms)Cleaning up temporary data from old regions at 1733268312829 (+2 ms)Region opened successfully at 1733268312835 (+6 ms) 2024-12-03T23:25:12,837 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 97eecf578d8a929685b4d07f76362d71, NAME => 'testDatalossWhenInputError,,1733268312279.97eecf578d8a929685b4d07f76362d71.', STARTKEY => '', ENDKEY => ''} 2024-12-03T23:25:12,838 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1733268312279.97eecf578d8a929685b4d07f76362d71.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T23:25:12,838 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 97eecf578d8a929685b4d07f76362d71 2024-12-03T23:25:12,838 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 97eecf578d8a929685b4d07f76362d71 2024-12-03T23:25:12,839 INFO [StoreOpener-97eecf578d8a929685b4d07f76362d71-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 97eecf578d8a929685b4d07f76362d71 2024-12-03T23:25:12,839 INFO [StoreOpener-97eecf578d8a929685b4d07f76362d71-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 97eecf578d8a929685b4d07f76362d71 columnFamilyName a 2024-12-03T23:25:12,840 DEBUG [StoreOpener-97eecf578d8a929685b4d07f76362d71-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T23:25:12,844 DEBUG [StoreOpener-97eecf578d8a929685b4d07f76362d71-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:35563/hbase/data/default/testDatalossWhenInputError/97eecf578d8a929685b4d07f76362d71/a/b571caecfed44af3b7cd36c419698ab8 2024-12-03T23:25:12,844 INFO [StoreOpener-97eecf578d8a929685b4d07f76362d71-1 {}] regionserver.HStore(327): Store=97eecf578d8a929685b4d07f76362d71/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T23:25:12,844 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 97eecf578d8a929685b4d07f76362d71 2024-12-03T23:25:12,845 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35563/hbase/data/default/testDatalossWhenInputError/97eecf578d8a929685b4d07f76362d71 2024-12-03T23:25:12,846 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35563/hbase/data/default/testDatalossWhenInputError/97eecf578d8a929685b4d07f76362d71 2024-12-03T23:25:12,846 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 97eecf578d8a929685b4d07f76362d71 2024-12-03T23:25:12,846 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 97eecf578d8a929685b4d07f76362d71 2024-12-03T23:25:12,847 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 97eecf578d8a929685b4d07f76362d71 2024-12-03T23:25:12,849 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35563/hbase/data/default/testDatalossWhenInputError/97eecf578d8a929685b4d07f76362d71/recovered.edits/13.seqid, newMaxSeqId=13, maxSeqId=12 2024-12-03T23:25:12,850 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 97eecf578d8a929685b4d07f76362d71; next sequenceid=14; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67965701, jitterRate=0.012767866253852844}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-03T23:25:12,850 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 97eecf578d8a929685b4d07f76362d71: Writing region info on filesystem at 1733268312838Initializing all the Stores at 1733268312838Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733268312838Cleaning up temporary data from old regions at 1733268312846 (+8 ms)Region opened successfully at 1733268312850 (+4 ms) 2024-12-03T23:25:12,867 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayCompressed#testDatalossWhenInputError Thread=430 (was 419) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-773788190_22 at /127.0.0.1:49730 [Receiving block BP-2057003906-172.17.0.2-1733268279817:blk_1073741918_1096] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-2057003906-172.17.0.2-1733268279817:blk_1073741918_1096, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-773788190_22 at /127.0.0.1:49672 [Waiting for operation #21] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-24-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-24-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-773788190_22 at /127.0.0.1:38226 [Receiving block BP-2057003906-172.17.0.2-1733268279817:blk_1073741918_1096] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-773788190_22 at /127.0.0.1:38246 [Waiting for operation #8] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-24-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-2057003906-172.17.0.2-1733268279817:blk_1073741918_1096, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-2057003906-172.17.0.2-1733268279817:blk_1073741918_1096, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-773788190_22 at /127.0.0.1:59106 [Waiting for operation #13] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-773788190_22 at /127.0.0.1:59188 [Receiving block BP-2057003906-172.17.0.2-1733268279817:blk_1073741918_1096] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1262 (was 1180) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=210 (was 210), ProcessCount=11 (was 11), AvailableMemoryMB=7800 (was 7811) 2024-12-03T23:25:12,867 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1262 is superior to 1024 2024-12-03T23:25:12,880 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayCompressed#testCompactedBulkLoadedFiles Thread=430, OpenFileDescriptor=1262, MaxFileDescriptor=1048576, SystemLoadAverage=210, ProcessCount=11, AvailableMemoryMB=7799 2024-12-03T23:25:12,880 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1262 is superior to 1024 2024-12-03T23:25:12,895 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-03T23:25:12,897 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-03T23:25:12,897 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-03T23:25:12,899 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-30959804, suffix=, logDir=hdfs://localhost:35563/hbase/WALs/hregion-30959804, archiveDir=hdfs://localhost:35563/hbase/oldWALs, maxLogs=32 2024-12-03T23:25:12,912 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-30959804/hregion-30959804.1733268312900, exclude list is [], retry=0 2024-12-03T23:25:12,914 DEBUG [AsyncFSWAL-26-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45491,DS-d55e5535-c0ce-4dc2-a45c-8e70eb010e56,DISK] 2024-12-03T23:25:12,914 DEBUG [AsyncFSWAL-26-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41487,DS-459d168f-61f7-403b-9139-91eca369c54e,DISK] 2024-12-03T23:25:12,915 DEBUG [AsyncFSWAL-26-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39829,DS-e6270720-ef81-458e-b5d6-959d16c4c378,DISK] 2024-12-03T23:25:12,917 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-30959804/hregion-30959804.1733268312900 2024-12-03T23:25:12,917 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34451:34451),(127.0.0.1/127.0.0.1:36361:36361),(127.0.0.1/127.0.0.1:38065:38065)] 2024-12-03T23:25:12,918 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => d56e940d9b532261bfbdd612e207397c, NAME => 'testCompactedBulkLoadedFiles,,1733268312895.d56e940d9b532261bfbdd612e207397c.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testCompactedBulkLoadedFiles', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35563/hbase 2024-12-03T23:25:12,925 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41487 is added to blk_1073741921_1099 (size=63) 2024-12-03T23:25:12,925 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39829 is added to blk_1073741921_1099 (size=63) 2024-12-03T23:25:12,926 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45491 is added to blk_1073741921_1099 (size=63) 2024-12-03T23:25:12,926 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testCompactedBulkLoadedFiles,,1733268312895.d56e940d9b532261bfbdd612e207397c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T23:25:12,928 INFO [StoreOpener-d56e940d9b532261bfbdd612e207397c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region d56e940d9b532261bfbdd612e207397c 2024-12-03T23:25:12,929 INFO [StoreOpener-d56e940d9b532261bfbdd612e207397c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d56e940d9b532261bfbdd612e207397c columnFamilyName a 2024-12-03T23:25:12,929 DEBUG [StoreOpener-d56e940d9b532261bfbdd612e207397c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T23:25:12,930 INFO [StoreOpener-d56e940d9b532261bfbdd612e207397c-1 {}] regionserver.HStore(327): Store=d56e940d9b532261bfbdd612e207397c/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T23:25:12,930 INFO [StoreOpener-d56e940d9b532261bfbdd612e207397c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region d56e940d9b532261bfbdd612e207397c 2024-12-03T23:25:12,931 INFO [StoreOpener-d56e940d9b532261bfbdd612e207397c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d56e940d9b532261bfbdd612e207397c columnFamilyName b 2024-12-03T23:25:12,931 DEBUG [StoreOpener-d56e940d9b532261bfbdd612e207397c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T23:25:12,932 INFO [StoreOpener-d56e940d9b532261bfbdd612e207397c-1 {}] regionserver.HStore(327): Store=d56e940d9b532261bfbdd612e207397c/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T23:25:12,932 INFO [StoreOpener-d56e940d9b532261bfbdd612e207397c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region d56e940d9b532261bfbdd612e207397c 2024-12-03T23:25:12,933 INFO [StoreOpener-d56e940d9b532261bfbdd612e207397c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d56e940d9b532261bfbdd612e207397c columnFamilyName c 2024-12-03T23:25:12,933 DEBUG [StoreOpener-d56e940d9b532261bfbdd612e207397c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T23:25:12,933 INFO [StoreOpener-d56e940d9b532261bfbdd612e207397c-1 {}] regionserver.HStore(327): Store=d56e940d9b532261bfbdd612e207397c/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T23:25:12,934 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for d56e940d9b532261bfbdd612e207397c 2024-12-03T23:25:12,934 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35563/hbase/data/default/testCompactedBulkLoadedFiles/d56e940d9b532261bfbdd612e207397c 2024-12-03T23:25:12,935 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35563/hbase/data/default/testCompactedBulkLoadedFiles/d56e940d9b532261bfbdd612e207397c 2024-12-03T23:25:12,936 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for d56e940d9b532261bfbdd612e207397c 2024-12-03T23:25:12,936 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for d56e940d9b532261bfbdd612e207397c 2024-12-03T23:25:12,936 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testCompactedBulkLoadedFiles descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-03T23:25:12,937 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for d56e940d9b532261bfbdd612e207397c 2024-12-03T23:25:12,939 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35563/hbase/data/default/testCompactedBulkLoadedFiles/d56e940d9b532261bfbdd612e207397c/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T23:25:12,940 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened d56e940d9b532261bfbdd612e207397c; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62586768, jitterRate=-0.06738448143005371}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-03T23:25:12,941 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for d56e940d9b532261bfbdd612e207397c: Writing region info on filesystem at 1733268312926Initializing all the Stores at 1733268312927 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733268312927Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733268312928 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733268312928Cleaning up temporary data from old regions at 1733268312936 (+8 ms)Region opened successfully at 1733268312941 (+5 ms) 2024-12-03T23:25:12,941 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing d56e940d9b532261bfbdd612e207397c, disabling compactions & flushes 2024-12-03T23:25:12,941 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testCompactedBulkLoadedFiles,,1733268312895.d56e940d9b532261bfbdd612e207397c. 2024-12-03T23:25:12,941 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testCompactedBulkLoadedFiles,,1733268312895.d56e940d9b532261bfbdd612e207397c. 2024-12-03T23:25:12,941 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testCompactedBulkLoadedFiles,,1733268312895.d56e940d9b532261bfbdd612e207397c. after waiting 0 ms 2024-12-03T23:25:12,941 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testCompactedBulkLoadedFiles,,1733268312895.d56e940d9b532261bfbdd612e207397c. 2024-12-03T23:25:12,941 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testCompactedBulkLoadedFiles,,1733268312895.d56e940d9b532261bfbdd612e207397c. 2024-12-03T23:25:12,941 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for d56e940d9b532261bfbdd612e207397c: Waiting for close lock at 1733268312941Disabling compacts and flushes for region at 1733268312941Disabling writes for close at 1733268312941Writing region close event to WAL at 1733268312941Closed at 1733268312941 2024-12-03T23:25:12,944 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41487 is added to blk_1073741920_1098 (size=93) 2024-12-03T23:25:12,944 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45491 is added to blk_1073741920_1098 (size=93) 2024-12-03T23:25:12,944 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39829 is added to blk_1073741920_1098 (size=93) 2024-12-03T23:25:12,947 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-03T23:25:12,947 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-30959804:(num 1733268312900) 2024-12-03T23:25:12,948 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-03T23:25:12,949 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:35563/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733268312894, archiveDir=hdfs://localhost:35563/hbase/oldWALs, maxLogs=32 2024-12-03T23:25:12,962 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733268312894/wal.1733268312949, exclude list is [], retry=0 2024-12-03T23:25:12,965 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45491,DS-d55e5535-c0ce-4dc2-a45c-8e70eb010e56,DISK] 2024-12-03T23:25:12,965 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39829,DS-e6270720-ef81-458e-b5d6-959d16c4c378,DISK] 2024-12-03T23:25:12,966 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41487,DS-459d168f-61f7-403b-9139-91eca369c54e,DISK] 2024-12-03T23:25:12,967 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733268312894/wal.1733268312949 2024-12-03T23:25:12,967 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34451:34451),(127.0.0.1/127.0.0.1:38065:38065),(127.0.0.1/127.0.0.1:36361:36361)] 2024-12-03T23:25:12,967 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => d56e940d9b532261bfbdd612e207397c, NAME => 'testCompactedBulkLoadedFiles,,1733268312895.d56e940d9b532261bfbdd612e207397c.', STARTKEY => '', ENDKEY => ''} 2024-12-03T23:25:12,968 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testCompactedBulkLoadedFiles,,1733268312895.d56e940d9b532261bfbdd612e207397c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T23:25:12,968 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for d56e940d9b532261bfbdd612e207397c 2024-12-03T23:25:12,968 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for d56e940d9b532261bfbdd612e207397c 2024-12-03T23:25:12,969 INFO [StoreOpener-d56e940d9b532261bfbdd612e207397c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region d56e940d9b532261bfbdd612e207397c 2024-12-03T23:25:12,970 INFO [StoreOpener-d56e940d9b532261bfbdd612e207397c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d56e940d9b532261bfbdd612e207397c columnFamilyName a 2024-12-03T23:25:12,970 DEBUG [StoreOpener-d56e940d9b532261bfbdd612e207397c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T23:25:12,971 INFO [StoreOpener-d56e940d9b532261bfbdd612e207397c-1 {}] regionserver.HStore(327): Store=d56e940d9b532261bfbdd612e207397c/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T23:25:12,971 INFO [StoreOpener-d56e940d9b532261bfbdd612e207397c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region d56e940d9b532261bfbdd612e207397c 2024-12-03T23:25:12,972 INFO [StoreOpener-d56e940d9b532261bfbdd612e207397c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d56e940d9b532261bfbdd612e207397c columnFamilyName b 2024-12-03T23:25:12,972 DEBUG [StoreOpener-d56e940d9b532261bfbdd612e207397c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T23:25:12,972 INFO [StoreOpener-d56e940d9b532261bfbdd612e207397c-1 {}] regionserver.HStore(327): Store=d56e940d9b532261bfbdd612e207397c/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T23:25:12,972 INFO [StoreOpener-d56e940d9b532261bfbdd612e207397c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region d56e940d9b532261bfbdd612e207397c 2024-12-03T23:25:12,973 INFO [StoreOpener-d56e940d9b532261bfbdd612e207397c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d56e940d9b532261bfbdd612e207397c columnFamilyName c 2024-12-03T23:25:12,973 DEBUG [StoreOpener-d56e940d9b532261bfbdd612e207397c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T23:25:12,974 INFO [StoreOpener-d56e940d9b532261bfbdd612e207397c-1 {}] regionserver.HStore(327): Store=d56e940d9b532261bfbdd612e207397c/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T23:25:12,974 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for d56e940d9b532261bfbdd612e207397c 2024-12-03T23:25:12,974 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35563/hbase/data/default/testCompactedBulkLoadedFiles/d56e940d9b532261bfbdd612e207397c 2024-12-03T23:25:12,975 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35563/hbase/data/default/testCompactedBulkLoadedFiles/d56e940d9b532261bfbdd612e207397c 2024-12-03T23:25:12,976 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for d56e940d9b532261bfbdd612e207397c 2024-12-03T23:25:12,976 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for d56e940d9b532261bfbdd612e207397c 2024-12-03T23:25:12,977 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testCompactedBulkLoadedFiles descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-03T23:25:12,978 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for d56e940d9b532261bfbdd612e207397c 2024-12-03T23:25:12,979 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened d56e940d9b532261bfbdd612e207397c; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59170774, jitterRate=-0.11828675866127014}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-03T23:25:12,980 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for d56e940d9b532261bfbdd612e207397c: Writing region info on filesystem at 1733268312968Initializing all the Stores at 1733268312969 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733268312969Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733268312969Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733268312969Cleaning up temporary data from old regions at 1733268312976 (+7 ms)Region opened successfully at 1733268312980 (+4 ms) 2024-12-03T23:25:12,984 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35563/hbase/testCompactedBulkLoadedFiles/hfile0 is 32, key is 000/a:a/1733268312984/Put/seqid=0 2024-12-03T23:25:12,994 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39829 is added to blk_1073741923_1101 (size=4875) 2024-12-03T23:25:12,995 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41487 is added to blk_1073741923_1101 (size=4875) 2024-12-03T23:25:12,995 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45491 is added to blk_1073741923_1101 (size=4875) 2024-12-03T23:25:12,998 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35563/hbase/testCompactedBulkLoadedFiles/hfile1 is 32, key is 100/a:a/1733268312997/Put/seqid=0 2024-12-03T23:25:13,010 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45491 is added to blk_1073741924_1102 (size=4875) 2024-12-03T23:25:13,010 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39829 is added to blk_1073741924_1102 (size=4875) 2024-12-03T23:25:13,010 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41487 is added to blk_1073741924_1102 (size=4875) 2024-12-03T23:25:13,012 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35563/hbase/testCompactedBulkLoadedFiles/hfile2 is 32, key is 200/a:a/1733268313012/Put/seqid=0 2024-12-03T23:25:13,018 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45491 is added to blk_1073741925_1103 (size=4875) 2024-12-03T23:25:13,018 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39829 is added to blk_1073741925_1103 (size=4875) 2024-12-03T23:25:13,018 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41487 is added to blk_1073741925_1103 (size=4875) 2024-12-03T23:25:13,019 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:35563/hbase/testCompactedBulkLoadedFiles/hfile0 for inclusion in d56e940d9b532261bfbdd612e207397c/a 2024-12-03T23:25:13,023 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first=000 last=050 2024-12-03T23:25:13,023 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-12-03T23:25:13,023 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:35563/hbase/testCompactedBulkLoadedFiles/hfile1 for inclusion in d56e940d9b532261bfbdd612e207397c/a 2024-12-03T23:25:13,027 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first=100 last=150 2024-12-03T23:25:13,027 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-12-03T23:25:13,027 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:35563/hbase/testCompactedBulkLoadedFiles/hfile2 for inclusion in d56e940d9b532261bfbdd612e207397c/a 2024-12-03T23:25:13,030 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first=200 last=250 2024-12-03T23:25:13,030 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-12-03T23:25:13,030 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing d56e940d9b532261bfbdd612e207397c 3/3 column families, dataSize=51 B heapSize=896 B 2024-12-03T23:25:13,050 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35563/hbase/data/default/testCompactedBulkLoadedFiles/d56e940d9b532261bfbdd612e207397c/.tmp/a/eec8d9a353964f2780a31282db5a63c1 is 55, key is testCompactedBulkLoadedFiles/a:a/1733268312980/Put/seqid=0 2024-12-03T23:25:13,056 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41487 is added to blk_1073741926_1104 (size=5107) 2024-12-03T23:25:13,056 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39829 is added to blk_1073741926_1104 (size=5107) 2024-12-03T23:25:13,056 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45491 is added to blk_1073741926_1104 (size=5107) 2024-12-03T23:25:13,057 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51 B at sequenceid=4 (bloomFilter=true), to=hdfs://localhost:35563/hbase/data/default/testCompactedBulkLoadedFiles/d56e940d9b532261bfbdd612e207397c/.tmp/a/eec8d9a353964f2780a31282db5a63c1 2024-12-03T23:25:13,064 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35563/hbase/data/default/testCompactedBulkLoadedFiles/d56e940d9b532261bfbdd612e207397c/.tmp/a/eec8d9a353964f2780a31282db5a63c1 as hdfs://localhost:35563/hbase/data/default/testCompactedBulkLoadedFiles/d56e940d9b532261bfbdd612e207397c/a/eec8d9a353964f2780a31282db5a63c1 2024-12-03T23:25:13,069 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35563/hbase/data/default/testCompactedBulkLoadedFiles/d56e940d9b532261bfbdd612e207397c/a/eec8d9a353964f2780a31282db5a63c1, entries=1, sequenceid=4, filesize=5.0 K 2024-12-03T23:25:13,070 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~51 B/51, heapSize ~368 B/368, currentSize=0 B/0 for d56e940d9b532261bfbdd612e207397c in 40ms, sequenceid=4, compaction requested=false 2024-12-03T23:25:13,070 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for d56e940d9b532261bfbdd612e207397c: 2024-12-03T23:25:13,072 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35563/hbase/testCompactedBulkLoadedFiles/hfile0 as hdfs://localhost:35563/hbase/data/default/testCompactedBulkLoadedFiles/d56e940d9b532261bfbdd612e207397c/a/70fcd3f1f6904fb3ae7654511c8c38ad_SeqId_4_ 2024-12-03T23:25:13,073 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35563/hbase/testCompactedBulkLoadedFiles/hfile1 as hdfs://localhost:35563/hbase/data/default/testCompactedBulkLoadedFiles/d56e940d9b532261bfbdd612e207397c/a/48806068421a4c60886e63a122094b66_SeqId_4_ 2024-12-03T23:25:13,074 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35563/hbase/testCompactedBulkLoadedFiles/hfile2 as hdfs://localhost:35563/hbase/data/default/testCompactedBulkLoadedFiles/d56e940d9b532261bfbdd612e207397c/a/2d849fa345854f67a3b5ddecb23cf385_SeqId_4_ 2024-12-03T23:25:13,074 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:35563/hbase/testCompactedBulkLoadedFiles/hfile0 into d56e940d9b532261bfbdd612e207397c/a as hdfs://localhost:35563/hbase/data/default/testCompactedBulkLoadedFiles/d56e940d9b532261bfbdd612e207397c/a/70fcd3f1f6904fb3ae7654511c8c38ad_SeqId_4_ - updating store file list. 2024-12-03T23:25:13,078 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for 70fcd3f1f6904fb3ae7654511c8c38ad_SeqId_4_: NONE, but ROW specified in column family configuration 2024-12-03T23:25:13,079 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:35563/hbase/data/default/testCompactedBulkLoadedFiles/d56e940d9b532261bfbdd612e207397c/a/70fcd3f1f6904fb3ae7654511c8c38ad_SeqId_4_ into d56e940d9b532261bfbdd612e207397c/a 2024-12-03T23:25:13,079 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:35563/hbase/testCompactedBulkLoadedFiles/hfile0 into d56e940d9b532261bfbdd612e207397c/a (new location: hdfs://localhost:35563/hbase/data/default/testCompactedBulkLoadedFiles/d56e940d9b532261bfbdd612e207397c/a/70fcd3f1f6904fb3ae7654511c8c38ad_SeqId_4_) 2024-12-03T23:25:13,079 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:35563/hbase/testCompactedBulkLoadedFiles/hfile1 into d56e940d9b532261bfbdd612e207397c/a as hdfs://localhost:35563/hbase/data/default/testCompactedBulkLoadedFiles/d56e940d9b532261bfbdd612e207397c/a/48806068421a4c60886e63a122094b66_SeqId_4_ - updating store file list. 2024-12-03T23:25:13,083 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for 48806068421a4c60886e63a122094b66_SeqId_4_: NONE, but ROW specified in column family configuration 2024-12-03T23:25:13,084 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:35563/hbase/data/default/testCompactedBulkLoadedFiles/d56e940d9b532261bfbdd612e207397c/a/48806068421a4c60886e63a122094b66_SeqId_4_ into d56e940d9b532261bfbdd612e207397c/a 2024-12-03T23:25:13,084 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:35563/hbase/testCompactedBulkLoadedFiles/hfile1 into d56e940d9b532261bfbdd612e207397c/a (new location: hdfs://localhost:35563/hbase/data/default/testCompactedBulkLoadedFiles/d56e940d9b532261bfbdd612e207397c/a/48806068421a4c60886e63a122094b66_SeqId_4_) 2024-12-03T23:25:13,084 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:35563/hbase/testCompactedBulkLoadedFiles/hfile2 into d56e940d9b532261bfbdd612e207397c/a as hdfs://localhost:35563/hbase/data/default/testCompactedBulkLoadedFiles/d56e940d9b532261bfbdd612e207397c/a/2d849fa345854f67a3b5ddecb23cf385_SeqId_4_ - updating store file list. 2024-12-03T23:25:13,088 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for 2d849fa345854f67a3b5ddecb23cf385_SeqId_4_: NONE, but ROW specified in column family configuration 2024-12-03T23:25:13,088 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:35563/hbase/data/default/testCompactedBulkLoadedFiles/d56e940d9b532261bfbdd612e207397c/a/2d849fa345854f67a3b5ddecb23cf385_SeqId_4_ into d56e940d9b532261bfbdd612e207397c/a 2024-12-03T23:25:13,088 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:35563/hbase/testCompactedBulkLoadedFiles/hfile2 into d56e940d9b532261bfbdd612e207397c/a (new location: hdfs://localhost:35563/hbase/data/default/testCompactedBulkLoadedFiles/d56e940d9b532261bfbdd612e207397c/a/2d849fa345854f67a3b5ddecb23cf385_SeqId_4_) 2024-12-03T23:25:13,096 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T23:25:13,096 DEBUG [Time-limited test {}] regionserver.HStore(1541): d56e940d9b532261bfbdd612e207397c/a is initiating major compaction (all files) 2024-12-03T23:25:13,096 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of d56e940d9b532261bfbdd612e207397c/a in testCompactedBulkLoadedFiles,,1733268312895.d56e940d9b532261bfbdd612e207397c. 2024-12-03T23:25:13,096 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35563/hbase/data/default/testCompactedBulkLoadedFiles/d56e940d9b532261bfbdd612e207397c/a/eec8d9a353964f2780a31282db5a63c1, hdfs://localhost:35563/hbase/data/default/testCompactedBulkLoadedFiles/d56e940d9b532261bfbdd612e207397c/a/70fcd3f1f6904fb3ae7654511c8c38ad_SeqId_4_, hdfs://localhost:35563/hbase/data/default/testCompactedBulkLoadedFiles/d56e940d9b532261bfbdd612e207397c/a/48806068421a4c60886e63a122094b66_SeqId_4_, hdfs://localhost:35563/hbase/data/default/testCompactedBulkLoadedFiles/d56e940d9b532261bfbdd612e207397c/a/2d849fa345854f67a3b5ddecb23cf385_SeqId_4_] into tmpdir=hdfs://localhost:35563/hbase/data/default/testCompactedBulkLoadedFiles/d56e940d9b532261bfbdd612e207397c/.tmp, totalSize=19.3 K 2024-12-03T23:25:13,097 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting eec8d9a353964f2780a31282db5a63c1, keycount=1, bloomtype=ROW, size=5.0 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=1733268312980 2024-12-03T23:25:13,097 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 70fcd3f1f6904fb3ae7654511c8c38ad_SeqId_4_, keycount=10, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=-9223372036854775808 2024-12-03T23:25:13,097 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 48806068421a4c60886e63a122094b66_SeqId_4_, keycount=10, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=-9223372036854775808 2024-12-03T23:25:13,098 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 2d849fa345854f67a3b5ddecb23cf385_SeqId_4_, keycount=10, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=-9223372036854775808 2024-12-03T23:25:13,110 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35563/hbase/data/default/testCompactedBulkLoadedFiles/d56e940d9b532261bfbdd612e207397c/.tmp/a/1d53bb53768b416b86ece434a2f3cb48 is 55, key is testCompactedBulkLoadedFiles/a:a/1733268312980/Put/seqid=0 2024-12-03T23:25:13,116 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45491 is added to blk_1073741927_1105 (size=6154) 2024-12-03T23:25:13,116 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39829 is added to blk_1073741927_1105 (size=6154) 2024-12-03T23:25:13,116 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41487 is added to blk_1073741927_1105 (size=6154) 2024-12-03T23:25:13,122 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35563/hbase/data/default/testCompactedBulkLoadedFiles/d56e940d9b532261bfbdd612e207397c/.tmp/a/1d53bb53768b416b86ece434a2f3cb48 as hdfs://localhost:35563/hbase/data/default/testCompactedBulkLoadedFiles/d56e940d9b532261bfbdd612e207397c/a/1d53bb53768b416b86ece434a2f3cb48 2024-12-03T23:25:13,127 INFO [Time-limited test {}] regionserver.HStore(1337): Completed major compaction of 4 (all) file(s) in d56e940d9b532261bfbdd612e207397c/a of d56e940d9b532261bfbdd612e207397c into 1d53bb53768b416b86ece434a2f3cb48(size=6.0 K), total size for store is 6.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T23:25:13,127 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for d56e940d9b532261bfbdd612e207397c: 2024-12-03T23:25:13,127 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 0 store files, 0 compacting, 0 eligible, 16 blocking 2024-12-03T23:25:13,127 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 0 store files, 0 compacting, 0 eligible, 16 blocking 2024-12-03T23:25:13,159 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:35563/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733268312894/wal.1733268312949, size=0 (0bytes) 2024-12-03T23:25:13,159 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:35563/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733268312894/wal.1733268312949 might be still open, length is 0 2024-12-03T23:25:13,159 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:35563/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733268312894/wal.1733268312949 2024-12-03T23:25:13,160 WARN [IPC Server handler 0 on default port 35563 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733268312894/wal.1733268312949 has not been closed. Lease recovery is in progress. RecoveryId = 1106 for block blk_1073741922_1100 2024-12-03T23:25:13,160 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:35563/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733268312894/wal.1733268312949 after 1ms 2024-12-03T23:25:13,258 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testDatalossWhenInputError 2024-12-03T23:25:13,258 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testDatalossWhenInputError Metrics about Tables on a single HBase RegionServer 2024-12-03T23:25:13,259 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testCompactedBulkLoadedFiles 2024-12-03T23:25:13,259 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testCompactedBulkLoadedFiles Metrics about Tables on a single HBase RegionServer 2024-12-03T23:25:14,772 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-773788190_22 at /127.0.0.1:38282 [Receiving block BP-2057003906-172.17.0.2-1733268279817:blk_1073741922_1100] {}] datanode.DataXceiver(331): 127.0.0.1:45491:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:38282 dst: /127.0.0.1:45491 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:45491 remote=/127.0.0.1:38282]. Total timeout mills is 60000, 58355 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T23:25:14,773 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-773788190_22 at /127.0.0.1:59204 [Receiving block BP-2057003906-172.17.0.2-1733268279817:blk_1073741922_1100] {}] datanode.DataXceiver(331): 127.0.0.1:41487:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:59204 dst: /127.0.0.1:41487 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T23:25:14,773 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-773788190_22 at /127.0.0.1:49758 [Receiving block BP-2057003906-172.17.0.2-1733268279817:blk_1073741922_1100] {}] datanode.DataXceiver(331): 127.0.0.1:39829:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:49758 dst: /127.0.0.1:39829 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T23:25:14,780 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45491 is added to blk_1073741922_1106 (size=1368) 2024-12-03T23:25:14,780 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39829 is added to blk_1073741922_1106 (size=1368) 2024-12-03T23:25:17,161 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:35563/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733268312894/wal.1733268312949 after 4002ms 2024-12-03T23:25:17,163 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:35563/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733268312894/wal.1733268312949: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-03T23:25:17,164 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:35563/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733268312894/wal.1733268312949 took 4005ms 2024-12-03T23:25:17,165 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:35563/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733268312894/wal.1733268312949; continuing. 2024-12-03T23:25:17,165 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:35563/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733268312894/wal.1733268312949 so closing down 2024-12-03T23:25:17,165 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-03T23:25:17,167 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1733268312949.temp 2024-12-03T23:25:17,168 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:35563/hbase/data/default/testCompactedBulkLoadedFiles/d56e940d9b532261bfbdd612e207397c/recovered.edits/0000000000000000003-wal.1733268312949.temp 2024-12-03T23:25:17,168 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-03T23:25:17,174 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45491 is added to blk_1073741928_1107 (size=635) 2024-12-03T23:25:17,174 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41487 is added to blk_1073741928_1107 (size=635) 2024-12-03T23:25:17,174 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39829 is added to blk_1073741928_1107 (size=635) 2024-12-03T23:25:17,175 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:35563/hbase/data/default/testCompactedBulkLoadedFiles/d56e940d9b532261bfbdd612e207397c/recovered.edits/0000000000000000003-wal.1733268312949.temp (wrote 2 edits, skipped 0 edits in 0 ms) 2024-12-03T23:25:17,176 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:35563/hbase/data/default/testCompactedBulkLoadedFiles/d56e940d9b532261bfbdd612e207397c/recovered.edits/0000000000000000003-wal.1733268312949.temp to hdfs://localhost:35563/hbase/data/default/testCompactedBulkLoadedFiles/d56e940d9b532261bfbdd612e207397c/recovered.edits/0000000000000000008 2024-12-03T23:25:17,176 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 5 edits across 1 Regions in 12 ms; skipped=3; WAL=hdfs://localhost:35563/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733268312894/wal.1733268312949, size=0, length=0, corrupted=false, cancelled=false 2024-12-03T23:25:17,176 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:35563/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733268312894/wal.1733268312949, journal: Splitting hdfs://localhost:35563/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733268312894/wal.1733268312949, size=0 (0bytes) at 1733268313159Finishing writing output for hdfs://localhost:35563/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733268312894/wal.1733268312949 so closing down at 1733268317165 (+4006 ms)Creating recovered edits writer path=hdfs://localhost:35563/hbase/data/default/testCompactedBulkLoadedFiles/d56e940d9b532261bfbdd612e207397c/recovered.edits/0000000000000000003-wal.1733268312949.temp at 1733268317168 (+3 ms)3 split writer threads finished at 1733268317168Closed recovered edits writer path=hdfs://localhost:35563/hbase/data/default/testCompactedBulkLoadedFiles/d56e940d9b532261bfbdd612e207397c/recovered.edits/0000000000000000003-wal.1733268312949.temp (wrote 2 edits, skipped 0 edits in 0 ms) at 1733268317175 (+7 ms)Rename recovered edits hdfs://localhost:35563/hbase/data/default/testCompactedBulkLoadedFiles/d56e940d9b532261bfbdd612e207397c/recovered.edits/0000000000000000003-wal.1733268312949.temp to hdfs://localhost:35563/hbase/data/default/testCompactedBulkLoadedFiles/d56e940d9b532261bfbdd612e207397c/recovered.edits/0000000000000000008 at 1733268317176 (+1 ms)Processed 5 edits across 1 Regions in 12 ms; skipped=3; WAL=hdfs://localhost:35563/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733268312894/wal.1733268312949, size=0, length=0, corrupted=false, cancelled=false at 1733268317176 2024-12-03T23:25:17,178 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:35563/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733268312894/wal.1733268312949 to hdfs://localhost:35563/hbase/oldWALs/wal.1733268312949 2024-12-03T23:25:17,178 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:35563/hbase/data/default/testCompactedBulkLoadedFiles/d56e940d9b532261bfbdd612e207397c/recovered.edits/0000000000000000008 2024-12-03T23:25:17,178 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-03T23:25:17,180 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:35563/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733268312894, archiveDir=hdfs://localhost:35563/hbase/oldWALs, maxLogs=32 2024-12-03T23:25:17,193 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733268312894/wal.1733268317180, exclude list is [], retry=0 2024-12-03T23:25:17,195 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41487,DS-459d168f-61f7-403b-9139-91eca369c54e,DISK] 2024-12-03T23:25:17,195 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45491,DS-d55e5535-c0ce-4dc2-a45c-8e70eb010e56,DISK] 2024-12-03T23:25:17,196 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39829,DS-e6270720-ef81-458e-b5d6-959d16c4c378,DISK] 2024-12-03T23:25:17,197 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733268312894/wal.1733268317180 2024-12-03T23:25:17,197 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36361:36361),(127.0.0.1/127.0.0.1:34451:34451),(127.0.0.1/127.0.0.1:38065:38065)] 2024-12-03T23:25:17,197 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => d56e940d9b532261bfbdd612e207397c, NAME => 'testCompactedBulkLoadedFiles,,1733268312895.d56e940d9b532261bfbdd612e207397c.', STARTKEY => '', ENDKEY => ''} 2024-12-03T23:25:17,198 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testCompactedBulkLoadedFiles,,1733268312895.d56e940d9b532261bfbdd612e207397c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T23:25:17,198 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for d56e940d9b532261bfbdd612e207397c 2024-12-03T23:25:17,198 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for d56e940d9b532261bfbdd612e207397c 2024-12-03T23:25:17,199 INFO [StoreOpener-d56e940d9b532261bfbdd612e207397c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region d56e940d9b532261bfbdd612e207397c 2024-12-03T23:25:17,200 INFO [StoreOpener-d56e940d9b532261bfbdd612e207397c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d56e940d9b532261bfbdd612e207397c columnFamilyName a 2024-12-03T23:25:17,200 DEBUG [StoreOpener-d56e940d9b532261bfbdd612e207397c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T23:25:17,206 DEBUG [StoreOpener-d56e940d9b532261bfbdd612e207397c-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:35563/hbase/data/default/testCompactedBulkLoadedFiles/d56e940d9b532261bfbdd612e207397c/a/1d53bb53768b416b86ece434a2f3cb48 2024-12-03T23:25:17,210 DEBUG [StoreFileOpener-d56e940d9b532261bfbdd612e207397c-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for 2d849fa345854f67a3b5ddecb23cf385_SeqId_4_: NONE, but ROW specified in column family configuration 2024-12-03T23:25:17,210 DEBUG [StoreOpener-d56e940d9b532261bfbdd612e207397c-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:35563/hbase/data/default/testCompactedBulkLoadedFiles/d56e940d9b532261bfbdd612e207397c/a/2d849fa345854f67a3b5ddecb23cf385_SeqId_4_ 2024-12-03T23:25:17,213 DEBUG [StoreFileOpener-d56e940d9b532261bfbdd612e207397c-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for 48806068421a4c60886e63a122094b66_SeqId_4_: NONE, but ROW specified in column family configuration 2024-12-03T23:25:17,213 DEBUG [StoreOpener-d56e940d9b532261bfbdd612e207397c-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:35563/hbase/data/default/testCompactedBulkLoadedFiles/d56e940d9b532261bfbdd612e207397c/a/48806068421a4c60886e63a122094b66_SeqId_4_ 2024-12-03T23:25:17,217 DEBUG [StoreFileOpener-d56e940d9b532261bfbdd612e207397c-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for 70fcd3f1f6904fb3ae7654511c8c38ad_SeqId_4_: NONE, but ROW specified in column family configuration 2024-12-03T23:25:17,217 DEBUG [StoreOpener-d56e940d9b532261bfbdd612e207397c-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:35563/hbase/data/default/testCompactedBulkLoadedFiles/d56e940d9b532261bfbdd612e207397c/a/70fcd3f1f6904fb3ae7654511c8c38ad_SeqId_4_ 2024-12-03T23:25:17,220 DEBUG [StoreOpener-d56e940d9b532261bfbdd612e207397c-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:35563/hbase/data/default/testCompactedBulkLoadedFiles/d56e940d9b532261bfbdd612e207397c/a/eec8d9a353964f2780a31282db5a63c1 2024-12-03T23:25:17,220 WARN [StoreOpener-d56e940d9b532261bfbdd612e207397c-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:35563/hbase/data/default/testCompactedBulkLoadedFiles/d56e940d9b532261bfbdd612e207397c/a/2d849fa345854f67a3b5ddecb23cf385_SeqId_4_ from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@9c032e9 2024-12-03T23:25:17,220 WARN [StoreOpener-d56e940d9b532261bfbdd612e207397c-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:35563/hbase/data/default/testCompactedBulkLoadedFiles/d56e940d9b532261bfbdd612e207397c/a/48806068421a4c60886e63a122094b66_SeqId_4_ from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@9c032e9 2024-12-03T23:25:17,220 WARN [StoreOpener-d56e940d9b532261bfbdd612e207397c-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:35563/hbase/data/default/testCompactedBulkLoadedFiles/d56e940d9b532261bfbdd612e207397c/a/70fcd3f1f6904fb3ae7654511c8c38ad_SeqId_4_ from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@9c032e9 2024-12-03T23:25:17,221 WARN [StoreOpener-d56e940d9b532261bfbdd612e207397c-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:35563/hbase/data/default/testCompactedBulkLoadedFiles/d56e940d9b532261bfbdd612e207397c/a/eec8d9a353964f2780a31282db5a63c1 from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@9c032e9 2024-12-03T23:25:17,221 DEBUG [StoreOpener-d56e940d9b532261bfbdd612e207397c-1 {}] regionserver.StoreEngine(327): Moving the files [hdfs://localhost:35563/hbase/data/default/testCompactedBulkLoadedFiles/d56e940d9b532261bfbdd612e207397c/a/2d849fa345854f67a3b5ddecb23cf385_SeqId_4_, hdfs://localhost:35563/hbase/data/default/testCompactedBulkLoadedFiles/d56e940d9b532261bfbdd612e207397c/a/48806068421a4c60886e63a122094b66_SeqId_4_, hdfs://localhost:35563/hbase/data/default/testCompactedBulkLoadedFiles/d56e940d9b532261bfbdd612e207397c/a/70fcd3f1f6904fb3ae7654511c8c38ad_SeqId_4_, hdfs://localhost:35563/hbase/data/default/testCompactedBulkLoadedFiles/d56e940d9b532261bfbdd612e207397c/a/eec8d9a353964f2780a31282db5a63c1] to archive 2024-12-03T23:25:17,221 DEBUG [StoreOpener-d56e940d9b532261bfbdd612e207397c-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-03T23:25:17,223 DEBUG [StoreOpener-d56e940d9b532261bfbdd612e207397c-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35563/hbase/data/default/testCompactedBulkLoadedFiles/d56e940d9b532261bfbdd612e207397c/a/2d849fa345854f67a3b5ddecb23cf385_SeqId_4_ to hdfs://localhost:35563/hbase/archive/data/default/testCompactedBulkLoadedFiles/d56e940d9b532261bfbdd612e207397c/a/2d849fa345854f67a3b5ddecb23cf385_SeqId_4_ 2024-12-03T23:25:17,224 DEBUG [StoreOpener-d56e940d9b532261bfbdd612e207397c-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35563/hbase/data/default/testCompactedBulkLoadedFiles/d56e940d9b532261bfbdd612e207397c/a/48806068421a4c60886e63a122094b66_SeqId_4_ to hdfs://localhost:35563/hbase/archive/data/default/testCompactedBulkLoadedFiles/d56e940d9b532261bfbdd612e207397c/a/48806068421a4c60886e63a122094b66_SeqId_4_ 2024-12-03T23:25:17,225 DEBUG [StoreOpener-d56e940d9b532261bfbdd612e207397c-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35563/hbase/data/default/testCompactedBulkLoadedFiles/d56e940d9b532261bfbdd612e207397c/a/70fcd3f1f6904fb3ae7654511c8c38ad_SeqId_4_ to hdfs://localhost:35563/hbase/archive/data/default/testCompactedBulkLoadedFiles/d56e940d9b532261bfbdd612e207397c/a/70fcd3f1f6904fb3ae7654511c8c38ad_SeqId_4_ 2024-12-03T23:25:17,226 DEBUG [StoreOpener-d56e940d9b532261bfbdd612e207397c-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35563/hbase/data/default/testCompactedBulkLoadedFiles/d56e940d9b532261bfbdd612e207397c/a/eec8d9a353964f2780a31282db5a63c1 to hdfs://localhost:35563/hbase/archive/data/default/testCompactedBulkLoadedFiles/d56e940d9b532261bfbdd612e207397c/a/eec8d9a353964f2780a31282db5a63c1 2024-12-03T23:25:17,226 INFO [StoreOpener-d56e940d9b532261bfbdd612e207397c-1 {}] regionserver.HStore(327): Store=d56e940d9b532261bfbdd612e207397c/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T23:25:17,226 INFO [StoreOpener-d56e940d9b532261bfbdd612e207397c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region d56e940d9b532261bfbdd612e207397c 2024-12-03T23:25:17,227 INFO [StoreOpener-d56e940d9b532261bfbdd612e207397c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d56e940d9b532261bfbdd612e207397c columnFamilyName b 2024-12-03T23:25:17,227 DEBUG [StoreOpener-d56e940d9b532261bfbdd612e207397c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T23:25:17,228 INFO [StoreOpener-d56e940d9b532261bfbdd612e207397c-1 {}] regionserver.HStore(327): Store=d56e940d9b532261bfbdd612e207397c/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T23:25:17,228 INFO [StoreOpener-d56e940d9b532261bfbdd612e207397c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region d56e940d9b532261bfbdd612e207397c 2024-12-03T23:25:17,229 INFO [StoreOpener-d56e940d9b532261bfbdd612e207397c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d56e940d9b532261bfbdd612e207397c columnFamilyName c 2024-12-03T23:25:17,229 DEBUG [StoreOpener-d56e940d9b532261bfbdd612e207397c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T23:25:17,229 INFO [StoreOpener-d56e940d9b532261bfbdd612e207397c-1 {}] regionserver.HStore(327): Store=d56e940d9b532261bfbdd612e207397c/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T23:25:17,229 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for d56e940d9b532261bfbdd612e207397c 2024-12-03T23:25:17,230 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35563/hbase/data/default/testCompactedBulkLoadedFiles/d56e940d9b532261bfbdd612e207397c 2024-12-03T23:25:17,231 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:35563/hbase/data/default/testCompactedBulkLoadedFiles/d56e940d9b532261bfbdd612e207397c 2024-12-03T23:25:17,232 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:35563/hbase/data/default/testCompactedBulkLoadedFiles/d56e940d9b532261bfbdd612e207397c/recovered.edits/0000000000000000008 2024-12-03T23:25:17,233 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:35563/hbase/data/default/testCompactedBulkLoadedFiles/d56e940d9b532261bfbdd612e207397c/recovered.edits/0000000000000000008: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-03T23:25:17,235 DEBUG [Time-limited test {}] regionserver.HRegion(5836): d56e940d9b532261bfbdd612e207397c : Replaying compaction marker table_name: "testCompactedBulkLoadedFiles" encoded_region_name: "d56e940d9b532261bfbdd612e207397c" family_name: "a" compaction_input: "eec8d9a353964f2780a31282db5a63c1" compaction_input: "70fcd3f1f6904fb3ae7654511c8c38ad_SeqId_4_" compaction_input: "48806068421a4c60886e63a122094b66_SeqId_4_" compaction_input: "2d849fa345854f67a3b5ddecb23cf385_SeqId_4_" compaction_output: "1d53bb53768b416b86ece434a2f3cb48" store_home_dir: "a" region_name: "testCompactedBulkLoadedFiles,,1733268312895.d56e940d9b532261bfbdd612e207397c." with seqId=9223372036854775807 and lastReplayedOpenRegionSeqId=-1 2024-12-03T23:25:17,235 DEBUG [Time-limited test {}] regionserver.HStore(1354): Completing compaction from the WAL marker 2024-12-03T23:25:17,235 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 0, skipped 2, firstSequenceIdInLog=3, maxSequenceIdInLog=8, path=hdfs://localhost:35563/hbase/data/default/testCompactedBulkLoadedFiles/d56e940d9b532261bfbdd612e207397c/recovered.edits/0000000000000000008 2024-12-03T23:25:17,235 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:35563/hbase/data/default/testCompactedBulkLoadedFiles/d56e940d9b532261bfbdd612e207397c/recovered.edits/0000000000000000008 2024-12-03T23:25:17,237 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for d56e940d9b532261bfbdd612e207397c 2024-12-03T23:25:17,237 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for d56e940d9b532261bfbdd612e207397c 2024-12-03T23:25:17,237 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testCompactedBulkLoadedFiles descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-03T23:25:17,239 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for d56e940d9b532261bfbdd612e207397c 2024-12-03T23:25:17,241 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35563/hbase/data/default/testCompactedBulkLoadedFiles/d56e940d9b532261bfbdd612e207397c/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-12-03T23:25:17,242 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened d56e940d9b532261bfbdd612e207397c; next sequenceid=9; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60433455, jitterRate=-0.09947134554386139}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-03T23:25:17,242 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for d56e940d9b532261bfbdd612e207397c: Writing region info on filesystem at 1733268317198Initializing all the Stores at 1733268317199 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733268317199Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733268317199Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733268317199Cleaning up temporary data from old regions at 1733268317237 (+38 ms)Region opened successfully at 1733268317242 (+5 ms) 2024-12-03T23:25:17,244 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing d56e940d9b532261bfbdd612e207397c, disabling compactions & flushes 2024-12-03T23:25:17,245 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testCompactedBulkLoadedFiles,,1733268312895.d56e940d9b532261bfbdd612e207397c. 2024-12-03T23:25:17,245 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testCompactedBulkLoadedFiles,,1733268312895.d56e940d9b532261bfbdd612e207397c. 2024-12-03T23:25:17,245 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testCompactedBulkLoadedFiles,,1733268312895.d56e940d9b532261bfbdd612e207397c. after waiting 0 ms 2024-12-03T23:25:17,245 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testCompactedBulkLoadedFiles,,1733268312895.d56e940d9b532261bfbdd612e207397c. 2024-12-03T23:25:17,246 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testCompactedBulkLoadedFiles,,1733268312895.d56e940d9b532261bfbdd612e207397c. 2024-12-03T23:25:17,246 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for d56e940d9b532261bfbdd612e207397c: Waiting for close lock at 1733268317244Disabling compacts and flushes for region at 1733268317244Disabling writes for close at 1733268317245 (+1 ms)Writing region close event to WAL at 1733268317245Closed at 1733268317245 2024-12-03T23:25:17,248 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45491 is added to blk_1073741929_1108 (size=93) 2024-12-03T23:25:17,248 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41487 is added to blk_1073741929_1108 (size=93) 2024-12-03T23:25:17,248 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39829 is added to blk_1073741929_1108 (size=93) 2024-12-03T23:25:17,250 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-03T23:25:17,250 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1733268317180) 2024-12-03T23:25:17,266 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayCompressed#testCompactedBulkLoadedFiles Thread=442 (was 430) Potentially hanging thread: LeaseRenewer:jenkinstestCompactedBulkLoadedFiles@localhost:35563 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_894743319_22 at /127.0.0.1:59284 [Waiting for operation #13] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-26-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1987708253) connection to localhost/127.0.0.1:35563 from jenkinstestCompactedBulkLoadedFiles java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_894743319_22 at /127.0.0.1:49820 [Waiting for operation #5] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-26-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-26-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1346 (was 1262) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=209 (was 210), ProcessCount=11 (was 11), AvailableMemoryMB=7779 (was 7799) 2024-12-03T23:25:17,266 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1346 is superior to 1024 2024-12-03T23:25:17,279 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayCompressed#testReplayEditsWrittenViaHRegion Thread=442, OpenFileDescriptor=1346, MaxFileDescriptor=1048576, SystemLoadAverage=209, ProcessCount=11, AvailableMemoryMB=7778 2024-12-03T23:25:17,279 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1346 is superior to 1024 2024-12-03T23:25:17,294 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-03T23:25:17,295 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-03T23:25:17,296 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-03T23:25:17,298 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-98640103, suffix=, logDir=hdfs://localhost:35563/hbase/WALs/hregion-98640103, archiveDir=hdfs://localhost:35563/hbase/oldWALs, maxLogs=32 2024-12-03T23:25:17,311 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-98640103/hregion-98640103.1733268317298, exclude list is [], retry=0 2024-12-03T23:25:17,314 DEBUG [AsyncFSWAL-28-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39829,DS-e6270720-ef81-458e-b5d6-959d16c4c378,DISK] 2024-12-03T23:25:17,314 DEBUG [AsyncFSWAL-28-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41487,DS-459d168f-61f7-403b-9139-91eca369c54e,DISK] 2024-12-03T23:25:17,315 DEBUG [AsyncFSWAL-28-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45491,DS-d55e5535-c0ce-4dc2-a45c-8e70eb010e56,DISK] 2024-12-03T23:25:17,318 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-98640103/hregion-98640103.1733268317298 2024-12-03T23:25:17,318 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:38065:38065),(127.0.0.1/127.0.0.1:36361:36361),(127.0.0.1/127.0.0.1:34451:34451)] 2024-12-03T23:25:17,318 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 524a0cd670c2b1984aedb9de0eb885d1, NAME => 'testReplayEditsWrittenViaHRegion,,1733268317294.524a0cd670c2b1984aedb9de0eb885d1.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenViaHRegion', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35563/hbase 2024-12-03T23:25:17,328 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45491 is added to blk_1073741931_1110 (size=67) 2024-12-03T23:25:17,328 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39829 is added to blk_1073741931_1110 (size=67) 2024-12-03T23:25:17,328 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41487 is added to blk_1073741931_1110 (size=67) 2024-12-03T23:25:17,329 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1733268317294.524a0cd670c2b1984aedb9de0eb885d1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T23:25:17,330 INFO [StoreOpener-524a0cd670c2b1984aedb9de0eb885d1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 524a0cd670c2b1984aedb9de0eb885d1 2024-12-03T23:25:17,332 INFO [StoreOpener-524a0cd670c2b1984aedb9de0eb885d1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 524a0cd670c2b1984aedb9de0eb885d1 columnFamilyName a 2024-12-03T23:25:17,332 DEBUG [StoreOpener-524a0cd670c2b1984aedb9de0eb885d1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T23:25:17,332 INFO [StoreOpener-524a0cd670c2b1984aedb9de0eb885d1-1 {}] regionserver.HStore(327): Store=524a0cd670c2b1984aedb9de0eb885d1/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T23:25:17,332 INFO [StoreOpener-524a0cd670c2b1984aedb9de0eb885d1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 524a0cd670c2b1984aedb9de0eb885d1 2024-12-03T23:25:17,334 INFO [StoreOpener-524a0cd670c2b1984aedb9de0eb885d1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 524a0cd670c2b1984aedb9de0eb885d1 columnFamilyName b 2024-12-03T23:25:17,334 DEBUG [StoreOpener-524a0cd670c2b1984aedb9de0eb885d1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T23:25:17,334 INFO [StoreOpener-524a0cd670c2b1984aedb9de0eb885d1-1 {}] regionserver.HStore(327): Store=524a0cd670c2b1984aedb9de0eb885d1/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T23:25:17,334 INFO [StoreOpener-524a0cd670c2b1984aedb9de0eb885d1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 524a0cd670c2b1984aedb9de0eb885d1 2024-12-03T23:25:17,335 INFO [StoreOpener-524a0cd670c2b1984aedb9de0eb885d1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 524a0cd670c2b1984aedb9de0eb885d1 columnFamilyName c 2024-12-03T23:25:17,335 DEBUG [StoreOpener-524a0cd670c2b1984aedb9de0eb885d1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T23:25:17,336 INFO [StoreOpener-524a0cd670c2b1984aedb9de0eb885d1-1 {}] regionserver.HStore(327): Store=524a0cd670c2b1984aedb9de0eb885d1/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T23:25:17,336 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 524a0cd670c2b1984aedb9de0eb885d1 2024-12-03T23:25:17,337 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35563/hbase/data/default/testReplayEditsWrittenViaHRegion/524a0cd670c2b1984aedb9de0eb885d1 2024-12-03T23:25:17,337 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35563/hbase/data/default/testReplayEditsWrittenViaHRegion/524a0cd670c2b1984aedb9de0eb885d1 2024-12-03T23:25:17,339 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 524a0cd670c2b1984aedb9de0eb885d1 2024-12-03T23:25:17,339 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 524a0cd670c2b1984aedb9de0eb885d1 2024-12-03T23:25:17,339 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-03T23:25:17,340 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 524a0cd670c2b1984aedb9de0eb885d1 2024-12-03T23:25:17,343 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35563/hbase/data/default/testReplayEditsWrittenViaHRegion/524a0cd670c2b1984aedb9de0eb885d1/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T23:25:17,343 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 524a0cd670c2b1984aedb9de0eb885d1; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61467226, jitterRate=-0.0840669572353363}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-03T23:25:17,343 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 524a0cd670c2b1984aedb9de0eb885d1: Writing region info on filesystem at 1733268317329Initializing all the Stores at 1733268317330 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733268317330Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733268317330Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733268317330Cleaning up temporary data from old regions at 1733268317339 (+9 ms)Region opened successfully at 1733268317343 (+4 ms) 2024-12-03T23:25:17,343 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 524a0cd670c2b1984aedb9de0eb885d1, disabling compactions & flushes 2024-12-03T23:25:17,343 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1733268317294.524a0cd670c2b1984aedb9de0eb885d1. 2024-12-03T23:25:17,343 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1733268317294.524a0cd670c2b1984aedb9de0eb885d1. 2024-12-03T23:25:17,343 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1733268317294.524a0cd670c2b1984aedb9de0eb885d1. after waiting 0 ms 2024-12-03T23:25:17,343 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1733268317294.524a0cd670c2b1984aedb9de0eb885d1. 2024-12-03T23:25:17,344 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1733268317294.524a0cd670c2b1984aedb9de0eb885d1. 2024-12-03T23:25:17,344 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 524a0cd670c2b1984aedb9de0eb885d1: Waiting for close lock at 1733268317343Disabling compacts and flushes for region at 1733268317343Disabling writes for close at 1733268317343Writing region close event to WAL at 1733268317344 (+1 ms)Closed at 1733268317344 2024-12-03T23:25:17,347 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45491 is added to blk_1073741930_1109 (size=93) 2024-12-03T23:25:17,347 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41487 is added to blk_1073741930_1109 (size=93) 2024-12-03T23:25:17,347 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39829 is added to blk_1073741930_1109 (size=93) 2024-12-03T23:25:17,349 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-03T23:25:17,349 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-98640103:(num 1733268317298) 2024-12-03T23:25:17,349 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-03T23:25:17,351 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:35563/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733268317293, archiveDir=hdfs://localhost:35563/hbase/oldWALs, maxLogs=32 2024-12-03T23:25:17,364 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733268317293/wal.1733268317351, exclude list is [], retry=0 2024-12-03T23:25:17,367 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39829,DS-e6270720-ef81-458e-b5d6-959d16c4c378,DISK] 2024-12-03T23:25:17,367 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41487,DS-459d168f-61f7-403b-9139-91eca369c54e,DISK] 2024-12-03T23:25:17,367 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45491,DS-d55e5535-c0ce-4dc2-a45c-8e70eb010e56,DISK] 2024-12-03T23:25:17,369 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733268317293/wal.1733268317351 2024-12-03T23:25:17,369 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:38065:38065),(127.0.0.1/127.0.0.1:36361:36361),(127.0.0.1/127.0.0.1:34451:34451)] 2024-12-03T23:25:17,369 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 524a0cd670c2b1984aedb9de0eb885d1, NAME => 'testReplayEditsWrittenViaHRegion,,1733268317294.524a0cd670c2b1984aedb9de0eb885d1.', STARTKEY => '', ENDKEY => ''} 2024-12-03T23:25:17,369 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1733268317294.524a0cd670c2b1984aedb9de0eb885d1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T23:25:17,369 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 524a0cd670c2b1984aedb9de0eb885d1 2024-12-03T23:25:17,369 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 524a0cd670c2b1984aedb9de0eb885d1 2024-12-03T23:25:17,370 INFO [StoreOpener-524a0cd670c2b1984aedb9de0eb885d1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 524a0cd670c2b1984aedb9de0eb885d1 2024-12-03T23:25:17,371 INFO [StoreOpener-524a0cd670c2b1984aedb9de0eb885d1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 524a0cd670c2b1984aedb9de0eb885d1 columnFamilyName a 2024-12-03T23:25:17,371 DEBUG [StoreOpener-524a0cd670c2b1984aedb9de0eb885d1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T23:25:17,372 INFO [StoreOpener-524a0cd670c2b1984aedb9de0eb885d1-1 {}] regionserver.HStore(327): Store=524a0cd670c2b1984aedb9de0eb885d1/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T23:25:17,372 INFO [StoreOpener-524a0cd670c2b1984aedb9de0eb885d1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 524a0cd670c2b1984aedb9de0eb885d1 2024-12-03T23:25:17,373 INFO [StoreOpener-524a0cd670c2b1984aedb9de0eb885d1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 524a0cd670c2b1984aedb9de0eb885d1 columnFamilyName b 2024-12-03T23:25:17,373 DEBUG [StoreOpener-524a0cd670c2b1984aedb9de0eb885d1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T23:25:17,373 INFO [StoreOpener-524a0cd670c2b1984aedb9de0eb885d1-1 {}] regionserver.HStore(327): Store=524a0cd670c2b1984aedb9de0eb885d1/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T23:25:17,373 INFO [StoreOpener-524a0cd670c2b1984aedb9de0eb885d1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 524a0cd670c2b1984aedb9de0eb885d1 2024-12-03T23:25:17,374 INFO [StoreOpener-524a0cd670c2b1984aedb9de0eb885d1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 524a0cd670c2b1984aedb9de0eb885d1 columnFamilyName c 2024-12-03T23:25:17,374 DEBUG [StoreOpener-524a0cd670c2b1984aedb9de0eb885d1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T23:25:17,374 INFO [StoreOpener-524a0cd670c2b1984aedb9de0eb885d1-1 {}] regionserver.HStore(327): Store=524a0cd670c2b1984aedb9de0eb885d1/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T23:25:17,374 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 524a0cd670c2b1984aedb9de0eb885d1 2024-12-03T23:25:17,375 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35563/hbase/data/default/testReplayEditsWrittenViaHRegion/524a0cd670c2b1984aedb9de0eb885d1 2024-12-03T23:25:17,376 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35563/hbase/data/default/testReplayEditsWrittenViaHRegion/524a0cd670c2b1984aedb9de0eb885d1 2024-12-03T23:25:17,377 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 524a0cd670c2b1984aedb9de0eb885d1 2024-12-03T23:25:17,377 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 524a0cd670c2b1984aedb9de0eb885d1 2024-12-03T23:25:17,377 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-03T23:25:17,378 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 524a0cd670c2b1984aedb9de0eb885d1 2024-12-03T23:25:17,379 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 524a0cd670c2b1984aedb9de0eb885d1; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63582044, jitterRate=-0.05255371332168579}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-03T23:25:17,379 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 524a0cd670c2b1984aedb9de0eb885d1: Writing region info on filesystem at 1733268317369Initializing all the Stores at 1733268317370 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733268317370Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733268317370Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733268317370Cleaning up temporary data from old regions at 1733268317377 (+7 ms)Region opened successfully at 1733268317379 (+2 ms) 2024-12-03T23:25:17,387 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 524a0cd670c2b1984aedb9de0eb885d1 3/3 column families, dataSize=870 B heapSize=2.31 KB 2024-12-03T23:25:17,402 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35563/hbase/data/default/testReplayEditsWrittenViaHRegion/524a0cd670c2b1984aedb9de0eb885d1/.tmp/a/5ceb48c84e8f4f1c8326a5926b669173 is 91, key is testReplayEditsWrittenViaHRegion/a:x0/1733268317379/Put/seqid=0 2024-12-03T23:25:17,408 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41487 is added to blk_1073741933_1112 (size=5958) 2024-12-03T23:25:17,409 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45491 is added to blk_1073741933_1112 (size=5958) 2024-12-03T23:25:17,409 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39829 is added to blk_1073741933_1112 (size=5958) 2024-12-03T23:25:17,411 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:35563/hbase/data/default/testReplayEditsWrittenViaHRegion/524a0cd670c2b1984aedb9de0eb885d1/.tmp/a/5ceb48c84e8f4f1c8326a5926b669173 2024-12-03T23:25:17,416 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35563/hbase/data/default/testReplayEditsWrittenViaHRegion/524a0cd670c2b1984aedb9de0eb885d1/.tmp/a/5ceb48c84e8f4f1c8326a5926b669173 as hdfs://localhost:35563/hbase/data/default/testReplayEditsWrittenViaHRegion/524a0cd670c2b1984aedb9de0eb885d1/a/5ceb48c84e8f4f1c8326a5926b669173 2024-12-03T23:25:17,421 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35563/hbase/data/default/testReplayEditsWrittenViaHRegion/524a0cd670c2b1984aedb9de0eb885d1/a/5ceb48c84e8f4f1c8326a5926b669173, entries=10, sequenceid=13, filesize=5.8 K 2024-12-03T23:25:17,422 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~870 B/870, heapSize ~1.80 KB/1840, currentSize=0 B/0 for 524a0cd670c2b1984aedb9de0eb885d1 in 35ms, sequenceid=13, compaction requested=false 2024-12-03T23:25:17,422 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 524a0cd670c2b1984aedb9de0eb885d1: 2024-12-03T23:25:17,438 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 524a0cd670c2b1984aedb9de0eb885d1, disabling compactions & flushes 2024-12-03T23:25:17,438 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1733268317294.524a0cd670c2b1984aedb9de0eb885d1. 2024-12-03T23:25:17,438 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1733268317294.524a0cd670c2b1984aedb9de0eb885d1. 2024-12-03T23:25:17,438 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1733268317294.524a0cd670c2b1984aedb9de0eb885d1. after waiting 0 ms 2024-12-03T23:25:17,438 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1733268317294.524a0cd670c2b1984aedb9de0eb885d1. 2024-12-03T23:25:17,439 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 1740 in region testReplayEditsWrittenViaHRegion,,1733268317294.524a0cd670c2b1984aedb9de0eb885d1. 2024-12-03T23:25:17,439 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1733268317294.524a0cd670c2b1984aedb9de0eb885d1. 2024-12-03T23:25:17,439 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 524a0cd670c2b1984aedb9de0eb885d1: Waiting for close lock at 1733268317438Disabling compacts and flushes for region at 1733268317438Disabling writes for close at 1733268317438Writing region close event to WAL at 1733268317439 (+1 ms)Closed at 1733268317439 2024-12-03T23:25:17,441 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45491 is added to blk_1073741932_1111 (size=2805) 2024-12-03T23:25:17,442 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39829 is added to blk_1073741932_1111 (size=2805) 2024-12-03T23:25:17,442 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41487 is added to blk_1073741932_1111 (size=2805) 2024-12-03T23:25:17,465 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:35563/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733268317293/wal.1733268317351, size=2.7 K (2805bytes) 2024-12-03T23:25:17,465 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:35563/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733268317293/wal.1733268317351 2024-12-03T23:25:17,465 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:35563/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733268317293/wal.1733268317351 after 0ms 2024-12-03T23:25:17,467 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:35563/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733268317293/wal.1733268317351: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-03T23:25:17,468 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:35563/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733268317293/wal.1733268317351 took 4ms 2024-12-03T23:25:17,470 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:35563/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733268317293/wal.1733268317351 so closing down 2024-12-03T23:25:17,470 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-03T23:25:17,471 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1733268317351.temp 2024-12-03T23:25:17,472 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:35563/hbase/data/default/testReplayEditsWrittenViaHRegion/524a0cd670c2b1984aedb9de0eb885d1/recovered.edits/0000000000000000003-wal.1733268317351.temp 2024-12-03T23:25:17,472 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-03T23:25:17,478 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41487 is added to blk_1073741934_1113 (size=2312) 2024-12-03T23:25:17,478 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45491 is added to blk_1073741934_1113 (size=2312) 2024-12-03T23:25:17,479 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39829 is added to blk_1073741934_1113 (size=2312) 2024-12-03T23:25:17,479 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:35563/hbase/data/default/testReplayEditsWrittenViaHRegion/524a0cd670c2b1984aedb9de0eb885d1/recovered.edits/0000000000000000003-wal.1733268317351.temp (wrote 30 edits, skipped 0 edits in 0 ms) 2024-12-03T23:25:17,481 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:35563/hbase/data/default/testReplayEditsWrittenViaHRegion/524a0cd670c2b1984aedb9de0eb885d1/recovered.edits/0000000000000000003-wal.1733268317351.temp to hdfs://localhost:35563/hbase/data/default/testReplayEditsWrittenViaHRegion/524a0cd670c2b1984aedb9de0eb885d1/recovered.edits/0000000000000000035 2024-12-03T23:25:17,481 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 32 edits across 1 Regions in 13 ms; skipped=2; WAL=hdfs://localhost:35563/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733268317293/wal.1733268317351, size=2.7 K, length=2805, corrupted=false, cancelled=false 2024-12-03T23:25:17,481 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:35563/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733268317293/wal.1733268317351, journal: Splitting hdfs://localhost:35563/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733268317293/wal.1733268317351, size=2.7 K (2805bytes) at 1733268317465Finishing writing output for hdfs://localhost:35563/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733268317293/wal.1733268317351 so closing down at 1733268317470 (+5 ms)Creating recovered edits writer path=hdfs://localhost:35563/hbase/data/default/testReplayEditsWrittenViaHRegion/524a0cd670c2b1984aedb9de0eb885d1/recovered.edits/0000000000000000003-wal.1733268317351.temp at 1733268317472 (+2 ms)3 split writer threads finished at 1733268317472Closed recovered edits writer path=hdfs://localhost:35563/hbase/data/default/testReplayEditsWrittenViaHRegion/524a0cd670c2b1984aedb9de0eb885d1/recovered.edits/0000000000000000003-wal.1733268317351.temp (wrote 30 edits, skipped 0 edits in 0 ms) at 1733268317479 (+7 ms)Rename recovered edits hdfs://localhost:35563/hbase/data/default/testReplayEditsWrittenViaHRegion/524a0cd670c2b1984aedb9de0eb885d1/recovered.edits/0000000000000000003-wal.1733268317351.temp to hdfs://localhost:35563/hbase/data/default/testReplayEditsWrittenViaHRegion/524a0cd670c2b1984aedb9de0eb885d1/recovered.edits/0000000000000000035 at 1733268317481 (+2 ms)Processed 32 edits across 1 Regions in 13 ms; skipped=2; WAL=hdfs://localhost:35563/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733268317293/wal.1733268317351, size=2.7 K, length=2805, corrupted=false, cancelled=false at 1733268317481 2024-12-03T23:25:17,483 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:35563/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733268317293/wal.1733268317351 to hdfs://localhost:35563/hbase/oldWALs/wal.1733268317351 2024-12-03T23:25:17,484 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:35563/hbase/data/default/testReplayEditsWrittenViaHRegion/524a0cd670c2b1984aedb9de0eb885d1/recovered.edits/0000000000000000035 2024-12-03T23:25:17,484 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-03T23:25:17,486 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:35563/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733268317293, archiveDir=hdfs://localhost:35563/hbase/oldWALs, maxLogs=32 2024-12-03T23:25:17,499 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733268317293/wal.1733268317486, exclude list is [], retry=0 2024-12-03T23:25:17,501 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45491,DS-d55e5535-c0ce-4dc2-a45c-8e70eb010e56,DISK] 2024-12-03T23:25:17,501 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41487,DS-459d168f-61f7-403b-9139-91eca369c54e,DISK] 2024-12-03T23:25:17,502 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39829,DS-e6270720-ef81-458e-b5d6-959d16c4c378,DISK] 2024-12-03T23:25:17,503 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733268317293/wal.1733268317486 2024-12-03T23:25:17,504 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34451:34451),(127.0.0.1/127.0.0.1:36361:36361),(127.0.0.1/127.0.0.1:38065:38065)] 2024-12-03T23:25:17,504 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 524a0cd670c2b1984aedb9de0eb885d1, NAME => 'testReplayEditsWrittenViaHRegion,,1733268317294.524a0cd670c2b1984aedb9de0eb885d1.', STARTKEY => '', ENDKEY => ''} 2024-12-03T23:25:17,504 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1733268317294.524a0cd670c2b1984aedb9de0eb885d1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T23:25:17,504 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 524a0cd670c2b1984aedb9de0eb885d1 2024-12-03T23:25:17,504 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 524a0cd670c2b1984aedb9de0eb885d1 2024-12-03T23:25:17,506 INFO [StoreOpener-524a0cd670c2b1984aedb9de0eb885d1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 524a0cd670c2b1984aedb9de0eb885d1 2024-12-03T23:25:17,506 INFO [StoreOpener-524a0cd670c2b1984aedb9de0eb885d1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 524a0cd670c2b1984aedb9de0eb885d1 columnFamilyName a 2024-12-03T23:25:17,506 DEBUG [StoreOpener-524a0cd670c2b1984aedb9de0eb885d1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T23:25:17,509 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-03T23:25:17,512 DEBUG [StoreOpener-524a0cd670c2b1984aedb9de0eb885d1-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:35563/hbase/data/default/testReplayEditsWrittenViaHRegion/524a0cd670c2b1984aedb9de0eb885d1/a/5ceb48c84e8f4f1c8326a5926b669173 2024-12-03T23:25:17,513 INFO [StoreOpener-524a0cd670c2b1984aedb9de0eb885d1-1 {}] regionserver.HStore(327): Store=524a0cd670c2b1984aedb9de0eb885d1/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T23:25:17,513 INFO [StoreOpener-524a0cd670c2b1984aedb9de0eb885d1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 524a0cd670c2b1984aedb9de0eb885d1 2024-12-03T23:25:17,514 INFO [StoreOpener-524a0cd670c2b1984aedb9de0eb885d1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 524a0cd670c2b1984aedb9de0eb885d1 columnFamilyName b 2024-12-03T23:25:17,514 DEBUG [StoreOpener-524a0cd670c2b1984aedb9de0eb885d1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T23:25:17,514 INFO [StoreOpener-524a0cd670c2b1984aedb9de0eb885d1-1 {}] regionserver.HStore(327): Store=524a0cd670c2b1984aedb9de0eb885d1/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T23:25:17,514 INFO [StoreOpener-524a0cd670c2b1984aedb9de0eb885d1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 524a0cd670c2b1984aedb9de0eb885d1 2024-12-03T23:25:17,515 INFO [StoreOpener-524a0cd670c2b1984aedb9de0eb885d1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 524a0cd670c2b1984aedb9de0eb885d1 columnFamilyName c 2024-12-03T23:25:17,515 DEBUG [StoreOpener-524a0cd670c2b1984aedb9de0eb885d1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T23:25:17,515 INFO [StoreOpener-524a0cd670c2b1984aedb9de0eb885d1-1 {}] regionserver.HStore(327): Store=524a0cd670c2b1984aedb9de0eb885d1/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T23:25:17,516 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 524a0cd670c2b1984aedb9de0eb885d1 2024-12-03T23:25:17,516 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35563/hbase/data/default/testReplayEditsWrittenViaHRegion/524a0cd670c2b1984aedb9de0eb885d1 2024-12-03T23:25:17,517 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:35563/hbase/data/default/testReplayEditsWrittenViaHRegion/524a0cd670c2b1984aedb9de0eb885d1 2024-12-03T23:25:17,518 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:35563/hbase/data/default/testReplayEditsWrittenViaHRegion/524a0cd670c2b1984aedb9de0eb885d1/recovered.edits/0000000000000000035 2024-12-03T23:25:17,520 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:35563/hbase/data/default/testReplayEditsWrittenViaHRegion/524a0cd670c2b1984aedb9de0eb885d1/recovered.edits/0000000000000000035: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-03T23:25:17,521 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 20, skipped 10, firstSequenceIdInLog=3, maxSequenceIdInLog=35, path=hdfs://localhost:35563/hbase/data/default/testReplayEditsWrittenViaHRegion/524a0cd670c2b1984aedb9de0eb885d1/recovered.edits/0000000000000000035 2024-12-03T23:25:17,521 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 524a0cd670c2b1984aedb9de0eb885d1 3/3 column families, dataSize=1.70 KB heapSize=3.88 KB 2024-12-03T23:25:17,536 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35563/hbase/data/default/testReplayEditsWrittenViaHRegion/524a0cd670c2b1984aedb9de0eb885d1/.tmp/b/8755b337c37342de9c39c3243630d602 is 91, key is testReplayEditsWrittenViaHRegion/b:x0/1733268317422/Put/seqid=0 2024-12-03T23:25:17,543 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45491 is added to blk_1073741936_1115 (size=5958) 2024-12-03T23:25:17,544 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39829 is added to blk_1073741936_1115 (size=5958) 2024-12-03T23:25:17,545 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=35 (bloomFilter=true), to=hdfs://localhost:35563/hbase/data/default/testReplayEditsWrittenViaHRegion/524a0cd670c2b1984aedb9de0eb885d1/.tmp/b/8755b337c37342de9c39c3243630d602 2024-12-03T23:25:17,545 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41487 is added to blk_1073741936_1115 (size=5958) 2024-12-03T23:25:17,564 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35563/hbase/data/default/testReplayEditsWrittenViaHRegion/524a0cd670c2b1984aedb9de0eb885d1/.tmp/c/0511627e9c1e480e828a3aafaacddbd3 is 91, key is testReplayEditsWrittenViaHRegion/c:x0/1733268317429/Put/seqid=0 2024-12-03T23:25:17,569 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41487 is added to blk_1073741937_1116 (size=5958) 2024-12-03T23:25:17,569 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39829 is added to blk_1073741937_1116 (size=5958) 2024-12-03T23:25:17,570 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45491 is added to blk_1073741937_1116 (size=5958) 2024-12-03T23:25:17,570 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=35 (bloomFilter=true), to=hdfs://localhost:35563/hbase/data/default/testReplayEditsWrittenViaHRegion/524a0cd670c2b1984aedb9de0eb885d1/.tmp/c/0511627e9c1e480e828a3aafaacddbd3 2024-12-03T23:25:17,575 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35563/hbase/data/default/testReplayEditsWrittenViaHRegion/524a0cd670c2b1984aedb9de0eb885d1/.tmp/b/8755b337c37342de9c39c3243630d602 as hdfs://localhost:35563/hbase/data/default/testReplayEditsWrittenViaHRegion/524a0cd670c2b1984aedb9de0eb885d1/b/8755b337c37342de9c39c3243630d602 2024-12-03T23:25:17,580 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35563/hbase/data/default/testReplayEditsWrittenViaHRegion/524a0cd670c2b1984aedb9de0eb885d1/b/8755b337c37342de9c39c3243630d602, entries=10, sequenceid=35, filesize=5.8 K 2024-12-03T23:25:17,581 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35563/hbase/data/default/testReplayEditsWrittenViaHRegion/524a0cd670c2b1984aedb9de0eb885d1/.tmp/c/0511627e9c1e480e828a3aafaacddbd3 as hdfs://localhost:35563/hbase/data/default/testReplayEditsWrittenViaHRegion/524a0cd670c2b1984aedb9de0eb885d1/c/0511627e9c1e480e828a3aafaacddbd3 2024-12-03T23:25:17,585 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35563/hbase/data/default/testReplayEditsWrittenViaHRegion/524a0cd670c2b1984aedb9de0eb885d1/c/0511627e9c1e480e828a3aafaacddbd3, entries=10, sequenceid=35, filesize=5.8 K 2024-12-03T23:25:17,585 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~1.70 KB/1740, heapSize ~3.59 KB/3680, currentSize=0 B/0 for 524a0cd670c2b1984aedb9de0eb885d1 in 64ms, sequenceid=35, compaction requested=false; wal=null 2024-12-03T23:25:17,586 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:35563/hbase/data/default/testReplayEditsWrittenViaHRegion/524a0cd670c2b1984aedb9de0eb885d1/recovered.edits/0000000000000000035 2024-12-03T23:25:17,587 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 524a0cd670c2b1984aedb9de0eb885d1 2024-12-03T23:25:17,587 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 524a0cd670c2b1984aedb9de0eb885d1 2024-12-03T23:25:17,588 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-03T23:25:17,589 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 524a0cd670c2b1984aedb9de0eb885d1 2024-12-03T23:25:17,591 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35563/hbase/data/default/testReplayEditsWrittenViaHRegion/524a0cd670c2b1984aedb9de0eb885d1/recovered.edits/35.seqid, newMaxSeqId=35, maxSeqId=1 2024-12-03T23:25:17,591 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 524a0cd670c2b1984aedb9de0eb885d1; next sequenceid=36; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64454369, jitterRate=-0.039555057883262634}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-03T23:25:17,592 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 524a0cd670c2b1984aedb9de0eb885d1: Writing region info on filesystem at 1733268317504Initializing all the Stores at 1733268317505 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733268317505Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733268317505Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733268317505Obtaining lock to block concurrent updates at 1733268317521 (+16 ms)Preparing flush snapshotting stores in 524a0cd670c2b1984aedb9de0eb885d1 at 1733268317521Finished memstore snapshotting testReplayEditsWrittenViaHRegion,,1733268317294.524a0cd670c2b1984aedb9de0eb885d1., syncing WAL and waiting on mvcc, flushsize=dataSize=1740, getHeapSize=3920, getOffHeapSize=0, getCellsCount=20 at 1733268317521Flushing stores of testReplayEditsWrittenViaHRegion,,1733268317294.524a0cd670c2b1984aedb9de0eb885d1. at 1733268317521Flushing 524a0cd670c2b1984aedb9de0eb885d1/b: creating writer at 1733268317521Flushing 524a0cd670c2b1984aedb9de0eb885d1/b: appending metadata at 1733268317535 (+14 ms)Flushing 524a0cd670c2b1984aedb9de0eb885d1/b: closing flushed file at 1733268317536 (+1 ms)Flushing 524a0cd670c2b1984aedb9de0eb885d1/c: creating writer at 1733268317549 (+13 ms)Flushing 524a0cd670c2b1984aedb9de0eb885d1/c: appending metadata at 1733268317563 (+14 ms)Flushing 524a0cd670c2b1984aedb9de0eb885d1/c: closing flushed file at 1733268317563Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@67155eb9: reopening flushed file at 1733268317574 (+11 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@133d3b7d: reopening flushed file at 1733268317580 (+6 ms)Finished flush of dataSize ~1.70 KB/1740, heapSize ~3.59 KB/3680, currentSize=0 B/0 for 524a0cd670c2b1984aedb9de0eb885d1 in 64ms, sequenceid=35, compaction requested=false; wal=null at 1733268317585 (+5 ms)Cleaning up temporary data from old regions at 1733268317587 (+2 ms)Region opened successfully at 1733268317592 (+5 ms) 2024-12-03T23:25:17,659 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:35563/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733268317293/wal.1733268317486, size=0 (0bytes) 2024-12-03T23:25:17,659 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:35563/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733268317293/wal.1733268317486 might be still open, length is 0 2024-12-03T23:25:17,660 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:35563/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733268317293/wal.1733268317486 2024-12-03T23:25:17,660 WARN [IPC Server handler 4 on default port 35563 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733268317293/wal.1733268317486 has not been closed. Lease recovery is in progress. RecoveryId = 1117 for block blk_1073741935_1114 2024-12-03T23:25:17,660 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:35563/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733268317293/wal.1733268317486 after 0ms 2024-12-03T23:25:17,770 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-773788190_22 at /127.0.0.1:38432 [Receiving block BP-2057003906-172.17.0.2-1733268279817:blk_1073741935_1114] {}] datanode.DataXceiver(331): 127.0.0.1:45491:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:38432 dst: /127.0.0.1:45491 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:45491 remote=/127.0.0.1:38432]. Total timeout mills is 60000, 59846 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T23:25:17,771 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-773788190_22 at /127.0.0.1:49876 [Receiving block BP-2057003906-172.17.0.2-1733268279817:blk_1073741935_1114] {}] datanode.DataXceiver(331): 127.0.0.1:39829:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:49876 dst: /127.0.0.1:39829 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T23:25:17,771 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-773788190_22 at /127.0.0.1:59350 [Receiving block BP-2057003906-172.17.0.2-1733268279817:blk_1073741935_1114] {}] datanode.DataXceiver(331): 127.0.0.1:41487:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:59350 dst: /127.0.0.1:41487 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T23:25:17,773 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45491 is added to blk_1073741935_1117 (size=2304) 2024-12-03T23:25:17,774 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41487 is added to blk_1073741935_1117 (size=2304) 2024-12-03T23:25:17,774 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39829 is added to blk_1073741935_1117 (size=2304) 2024-12-03T23:25:21,661 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:35563/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733268317293/wal.1733268317486 after 4001ms 2024-12-03T23:25:21,663 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:35563/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733268317293/wal.1733268317486: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-03T23:25:21,664 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:35563/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733268317293/wal.1733268317486 took 4005ms 2024-12-03T23:25:21,666 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:35563/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733268317293/wal.1733268317486; continuing. 2024-12-03T23:25:21,666 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:35563/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733268317293/wal.1733268317486 so closing down 2024-12-03T23:25:21,666 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-03T23:25:21,667 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000037-wal.1733268317486.temp 2024-12-03T23:25:21,668 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:35563/hbase/data/default/testReplayEditsWrittenViaHRegion/524a0cd670c2b1984aedb9de0eb885d1/recovered.edits/0000000000000000037-wal.1733268317486.temp 2024-12-03T23:25:21,668 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-03T23:25:21,675 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45491 is added to blk_1073741938_1118 (size=2312) 2024-12-03T23:25:21,675 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41487 is added to blk_1073741938_1118 (size=2312) 2024-12-03T23:25:21,676 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39829 is added to blk_1073741938_1118 (size=2312) 2024-12-03T23:25:21,676 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:35563/hbase/data/default/testReplayEditsWrittenViaHRegion/524a0cd670c2b1984aedb9de0eb885d1/recovered.edits/0000000000000000037-wal.1733268317486.temp (wrote 30 edits, skipped 0 edits in 0 ms) 2024-12-03T23:25:21,677 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:35563/hbase/data/default/testReplayEditsWrittenViaHRegion/524a0cd670c2b1984aedb9de0eb885d1/recovered.edits/0000000000000000037-wal.1733268317486.temp to hdfs://localhost:35563/hbase/data/default/testReplayEditsWrittenViaHRegion/524a0cd670c2b1984aedb9de0eb885d1/recovered.edits/0000000000000000066 2024-12-03T23:25:21,677 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 30 edits across 1 Regions in 13 ms; skipped=0; WAL=hdfs://localhost:35563/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733268317293/wal.1733268317486, size=0, length=0, corrupted=false, cancelled=false 2024-12-03T23:25:21,678 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:35563/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733268317293/wal.1733268317486, journal: Splitting hdfs://localhost:35563/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733268317293/wal.1733268317486, size=0 (0bytes) at 1733268317659Finishing writing output for hdfs://localhost:35563/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733268317293/wal.1733268317486 so closing down at 1733268321666 (+4007 ms)Creating recovered edits writer path=hdfs://localhost:35563/hbase/data/default/testReplayEditsWrittenViaHRegion/524a0cd670c2b1984aedb9de0eb885d1/recovered.edits/0000000000000000037-wal.1733268317486.temp at 1733268321668 (+2 ms)3 split writer threads finished at 1733268321668Closed recovered edits writer path=hdfs://localhost:35563/hbase/data/default/testReplayEditsWrittenViaHRegion/524a0cd670c2b1984aedb9de0eb885d1/recovered.edits/0000000000000000037-wal.1733268317486.temp (wrote 30 edits, skipped 0 edits in 0 ms) at 1733268321676 (+8 ms)Rename recovered edits hdfs://localhost:35563/hbase/data/default/testReplayEditsWrittenViaHRegion/524a0cd670c2b1984aedb9de0eb885d1/recovered.edits/0000000000000000037-wal.1733268317486.temp to hdfs://localhost:35563/hbase/data/default/testReplayEditsWrittenViaHRegion/524a0cd670c2b1984aedb9de0eb885d1/recovered.edits/0000000000000000066 at 1733268321677 (+1 ms)Processed 30 edits across 1 Regions in 13 ms; skipped=0; WAL=hdfs://localhost:35563/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733268317293/wal.1733268317486, size=0, length=0, corrupted=false, cancelled=false at 1733268321677 2024-12-03T23:25:21,679 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:35563/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733268317293/wal.1733268317486 to hdfs://localhost:35563/hbase/oldWALs/wal.1733268317486 2024-12-03T23:25:21,680 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:35563/hbase/data/default/testReplayEditsWrittenViaHRegion/524a0cd670c2b1984aedb9de0eb885d1/recovered.edits/0000000000000000066 2024-12-03T23:25:21,680 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-03T23:25:21,681 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:35563/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733268317293, archiveDir=hdfs://localhost:35563/hbase/oldWALs, maxLogs=32 2024-12-03T23:25:21,695 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733268317293/wal.1733268321682, exclude list is [], retry=0 2024-12-03T23:25:21,697 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39829,DS-e6270720-ef81-458e-b5d6-959d16c4c378,DISK] 2024-12-03T23:25:21,697 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41487,DS-459d168f-61f7-403b-9139-91eca369c54e,DISK] 2024-12-03T23:25:21,697 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45491,DS-d55e5535-c0ce-4dc2-a45c-8e70eb010e56,DISK] 2024-12-03T23:25:21,699 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733268317293/wal.1733268321682 2024-12-03T23:25:21,699 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:38065:38065),(127.0.0.1/127.0.0.1:36361:36361),(127.0.0.1/127.0.0.1:34451:34451)] 2024-12-03T23:25:21,699 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1733268317294.524a0cd670c2b1984aedb9de0eb885d1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T23:25:21,700 INFO [StoreOpener-524a0cd670c2b1984aedb9de0eb885d1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 524a0cd670c2b1984aedb9de0eb885d1 2024-12-03T23:25:21,701 INFO [StoreOpener-524a0cd670c2b1984aedb9de0eb885d1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 524a0cd670c2b1984aedb9de0eb885d1 columnFamilyName a 2024-12-03T23:25:21,701 DEBUG [StoreOpener-524a0cd670c2b1984aedb9de0eb885d1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T23:25:21,707 DEBUG [StoreOpener-524a0cd670c2b1984aedb9de0eb885d1-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:35563/hbase/data/default/testReplayEditsWrittenViaHRegion/524a0cd670c2b1984aedb9de0eb885d1/a/5ceb48c84e8f4f1c8326a5926b669173 2024-12-03T23:25:21,707 INFO [StoreOpener-524a0cd670c2b1984aedb9de0eb885d1-1 {}] regionserver.HStore(327): Store=524a0cd670c2b1984aedb9de0eb885d1/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T23:25:21,707 INFO [StoreOpener-524a0cd670c2b1984aedb9de0eb885d1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 524a0cd670c2b1984aedb9de0eb885d1 2024-12-03T23:25:21,708 INFO [StoreOpener-524a0cd670c2b1984aedb9de0eb885d1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 524a0cd670c2b1984aedb9de0eb885d1 columnFamilyName b 2024-12-03T23:25:21,708 DEBUG [StoreOpener-524a0cd670c2b1984aedb9de0eb885d1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T23:25:21,712 DEBUG [StoreOpener-524a0cd670c2b1984aedb9de0eb885d1-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:35563/hbase/data/default/testReplayEditsWrittenViaHRegion/524a0cd670c2b1984aedb9de0eb885d1/b/8755b337c37342de9c39c3243630d602 2024-12-03T23:25:21,712 INFO [StoreOpener-524a0cd670c2b1984aedb9de0eb885d1-1 {}] regionserver.HStore(327): Store=524a0cd670c2b1984aedb9de0eb885d1/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T23:25:21,712 INFO [StoreOpener-524a0cd670c2b1984aedb9de0eb885d1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 524a0cd670c2b1984aedb9de0eb885d1 2024-12-03T23:25:21,713 INFO [StoreOpener-524a0cd670c2b1984aedb9de0eb885d1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 524a0cd670c2b1984aedb9de0eb885d1 columnFamilyName c 2024-12-03T23:25:21,713 DEBUG [StoreOpener-524a0cd670c2b1984aedb9de0eb885d1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T23:25:21,717 DEBUG [StoreOpener-524a0cd670c2b1984aedb9de0eb885d1-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:35563/hbase/data/default/testReplayEditsWrittenViaHRegion/524a0cd670c2b1984aedb9de0eb885d1/c/0511627e9c1e480e828a3aafaacddbd3 2024-12-03T23:25:21,718 INFO [StoreOpener-524a0cd670c2b1984aedb9de0eb885d1-1 {}] regionserver.HStore(327): Store=524a0cd670c2b1984aedb9de0eb885d1/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T23:25:21,718 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 524a0cd670c2b1984aedb9de0eb885d1 2024-12-03T23:25:21,718 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35563/hbase/data/default/testReplayEditsWrittenViaHRegion/524a0cd670c2b1984aedb9de0eb885d1 2024-12-03T23:25:21,720 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:35563/hbase/data/default/testReplayEditsWrittenViaHRegion/524a0cd670c2b1984aedb9de0eb885d1 2024-12-03T23:25:21,720 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:35563/hbase/data/default/testReplayEditsWrittenViaHRegion/524a0cd670c2b1984aedb9de0eb885d1/recovered.edits/0000000000000000066 2024-12-03T23:25:21,722 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:35563/hbase/data/default/testReplayEditsWrittenViaHRegion/524a0cd670c2b1984aedb9de0eb885d1/recovered.edits/0000000000000000066: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-03T23:25:21,727 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 30, skipped 0, firstSequenceIdInLog=37, maxSequenceIdInLog=66, path=hdfs://localhost:35563/hbase/data/default/testReplayEditsWrittenViaHRegion/524a0cd670c2b1984aedb9de0eb885d1/recovered.edits/0000000000000000066 2024-12-03T23:25:21,727 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 524a0cd670c2b1984aedb9de0eb885d1 3/3 column families, dataSize=2.55 KB heapSize=5.44 KB 2024-12-03T23:25:21,742 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35563/hbase/data/default/testReplayEditsWrittenViaHRegion/524a0cd670c2b1984aedb9de0eb885d1/.tmp/a/a6f7bc311ed84669846a21d4aed82ae4 is 91, key is testReplayEditsWrittenViaHRegion/a:y0/1733268317598/Put/seqid=0 2024-12-03T23:25:21,747 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39829 is added to blk_1073741940_1120 (size=5958) 2024-12-03T23:25:21,747 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41487 is added to blk_1073741940_1120 (size=5958) 2024-12-03T23:25:21,748 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45491 is added to blk_1073741940_1120 (size=5958) 2024-12-03T23:25:21,748 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:35563/hbase/data/default/testReplayEditsWrittenViaHRegion/524a0cd670c2b1984aedb9de0eb885d1/.tmp/a/a6f7bc311ed84669846a21d4aed82ae4 2024-12-03T23:25:21,766 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35563/hbase/data/default/testReplayEditsWrittenViaHRegion/524a0cd670c2b1984aedb9de0eb885d1/.tmp/b/fa796a3ee7694d97ae02e112e974b061 is 91, key is testReplayEditsWrittenViaHRegion/b:y0/1733268317604/Put/seqid=0 2024-12-03T23:25:21,771 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39829 is added to blk_1073741941_1121 (size=5958) 2024-12-03T23:25:21,771 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45491 is added to blk_1073741941_1121 (size=5958) 2024-12-03T23:25:21,771 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41487 is added to blk_1073741941_1121 (size=5958) 2024-12-03T23:25:21,772 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:35563/hbase/data/default/testReplayEditsWrittenViaHRegion/524a0cd670c2b1984aedb9de0eb885d1/.tmp/b/fa796a3ee7694d97ae02e112e974b061 2024-12-03T23:25:21,790 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35563/hbase/data/default/testReplayEditsWrittenViaHRegion/524a0cd670c2b1984aedb9de0eb885d1/.tmp/c/8630c84db6354990ab6530159ddc0e36 is 91, key is testReplayEditsWrittenViaHRegion/c:y0/1733268317610/Put/seqid=0 2024-12-03T23:25:21,795 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45491 is added to blk_1073741942_1122 (size=5958) 2024-12-03T23:25:21,796 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39829 is added to blk_1073741942_1122 (size=5958) 2024-12-03T23:25:21,796 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41487 is added to blk_1073741942_1122 (size=5958) 2024-12-03T23:25:21,796 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:35563/hbase/data/default/testReplayEditsWrittenViaHRegion/524a0cd670c2b1984aedb9de0eb885d1/.tmp/c/8630c84db6354990ab6530159ddc0e36 2024-12-03T23:25:21,801 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35563/hbase/data/default/testReplayEditsWrittenViaHRegion/524a0cd670c2b1984aedb9de0eb885d1/.tmp/a/a6f7bc311ed84669846a21d4aed82ae4 as hdfs://localhost:35563/hbase/data/default/testReplayEditsWrittenViaHRegion/524a0cd670c2b1984aedb9de0eb885d1/a/a6f7bc311ed84669846a21d4aed82ae4 2024-12-03T23:25:21,806 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35563/hbase/data/default/testReplayEditsWrittenViaHRegion/524a0cd670c2b1984aedb9de0eb885d1/a/a6f7bc311ed84669846a21d4aed82ae4, entries=10, sequenceid=66, filesize=5.8 K 2024-12-03T23:25:21,806 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35563/hbase/data/default/testReplayEditsWrittenViaHRegion/524a0cd670c2b1984aedb9de0eb885d1/.tmp/b/fa796a3ee7694d97ae02e112e974b061 as hdfs://localhost:35563/hbase/data/default/testReplayEditsWrittenViaHRegion/524a0cd670c2b1984aedb9de0eb885d1/b/fa796a3ee7694d97ae02e112e974b061 2024-12-03T23:25:21,811 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35563/hbase/data/default/testReplayEditsWrittenViaHRegion/524a0cd670c2b1984aedb9de0eb885d1/b/fa796a3ee7694d97ae02e112e974b061, entries=10, sequenceid=66, filesize=5.8 K 2024-12-03T23:25:21,811 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35563/hbase/data/default/testReplayEditsWrittenViaHRegion/524a0cd670c2b1984aedb9de0eb885d1/.tmp/c/8630c84db6354990ab6530159ddc0e36 as hdfs://localhost:35563/hbase/data/default/testReplayEditsWrittenViaHRegion/524a0cd670c2b1984aedb9de0eb885d1/c/8630c84db6354990ab6530159ddc0e36 2024-12-03T23:25:21,815 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35563/hbase/data/default/testReplayEditsWrittenViaHRegion/524a0cd670c2b1984aedb9de0eb885d1/c/8630c84db6354990ab6530159ddc0e36, entries=10, sequenceid=66, filesize=5.8 K 2024-12-03T23:25:21,815 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~2.55 KB/2610, heapSize ~5.39 KB/5520, currentSize=0 B/0 for 524a0cd670c2b1984aedb9de0eb885d1 in 88ms, sequenceid=66, compaction requested=false; wal=null 2024-12-03T23:25:21,816 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:35563/hbase/data/default/testReplayEditsWrittenViaHRegion/524a0cd670c2b1984aedb9de0eb885d1/recovered.edits/0000000000000000066 2024-12-03T23:25:21,817 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 524a0cd670c2b1984aedb9de0eb885d1 2024-12-03T23:25:21,817 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 524a0cd670c2b1984aedb9de0eb885d1 2024-12-03T23:25:21,817 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-03T23:25:21,819 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 524a0cd670c2b1984aedb9de0eb885d1 2024-12-03T23:25:21,821 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35563/hbase/data/default/testReplayEditsWrittenViaHRegion/524a0cd670c2b1984aedb9de0eb885d1/recovered.edits/66.seqid, newMaxSeqId=66, maxSeqId=35 2024-12-03T23:25:21,821 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 524a0cd670c2b1984aedb9de0eb885d1; next sequenceid=67; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60086538, jitterRate=-0.10464081168174744}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-03T23:25:21,822 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 524a0cd670c2b1984aedb9de0eb885d1: Writing region info on filesystem at 1733268321699Initializing all the Stores at 1733268321700 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733268321700Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733268321700Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733268321700Obtaining lock to block concurrent updates at 1733268321727 (+27 ms)Preparing flush snapshotting stores in 524a0cd670c2b1984aedb9de0eb885d1 at 1733268321727Finished memstore snapshotting testReplayEditsWrittenViaHRegion,,1733268317294.524a0cd670c2b1984aedb9de0eb885d1., syncing WAL and waiting on mvcc, flushsize=dataSize=2610, getHeapSize=5520, getOffHeapSize=0, getCellsCount=30 at 1733268321727Flushing stores of testReplayEditsWrittenViaHRegion,,1733268317294.524a0cd670c2b1984aedb9de0eb885d1. at 1733268321727Flushing 524a0cd670c2b1984aedb9de0eb885d1/a: creating writer at 1733268321727Flushing 524a0cd670c2b1984aedb9de0eb885d1/a: appending metadata at 1733268321741 (+14 ms)Flushing 524a0cd670c2b1984aedb9de0eb885d1/a: closing flushed file at 1733268321741Flushing 524a0cd670c2b1984aedb9de0eb885d1/b: creating writer at 1733268321752 (+11 ms)Flushing 524a0cd670c2b1984aedb9de0eb885d1/b: appending metadata at 1733268321765 (+13 ms)Flushing 524a0cd670c2b1984aedb9de0eb885d1/b: closing flushed file at 1733268321765Flushing 524a0cd670c2b1984aedb9de0eb885d1/c: creating writer at 1733268321776 (+11 ms)Flushing 524a0cd670c2b1984aedb9de0eb885d1/c: appending metadata at 1733268321789 (+13 ms)Flushing 524a0cd670c2b1984aedb9de0eb885d1/c: closing flushed file at 1733268321789Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@530d88fc: reopening flushed file at 1733268321801 (+12 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@62a56669: reopening flushed file at 1733268321806 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2db62c62: reopening flushed file at 1733268321811 (+5 ms)Finished flush of dataSize ~2.55 KB/2610, heapSize ~5.39 KB/5520, currentSize=0 B/0 for 524a0cd670c2b1984aedb9de0eb885d1 in 88ms, sequenceid=66, compaction requested=false; wal=null at 1733268321815 (+4 ms)Cleaning up temporary data from old regions at 1733268321817 (+2 ms)Region opened successfully at 1733268321822 (+5 ms) 2024-12-03T23:25:21,833 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 524a0cd670c2b1984aedb9de0eb885d1, disabling compactions & flushes 2024-12-03T23:25:21,833 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1733268317294.524a0cd670c2b1984aedb9de0eb885d1. 2024-12-03T23:25:21,833 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1733268317294.524a0cd670c2b1984aedb9de0eb885d1. 2024-12-03T23:25:21,833 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1733268317294.524a0cd670c2b1984aedb9de0eb885d1. after waiting 0 ms 2024-12-03T23:25:21,833 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1733268317294.524a0cd670c2b1984aedb9de0eb885d1. 2024-12-03T23:25:21,834 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1733268317294.524a0cd670c2b1984aedb9de0eb885d1. 2024-12-03T23:25:21,834 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 524a0cd670c2b1984aedb9de0eb885d1: Waiting for close lock at 1733268321833Disabling compacts and flushes for region at 1733268321833Disabling writes for close at 1733268321833Writing region close event to WAL at 1733268321834 (+1 ms)Closed at 1733268321834 2024-12-03T23:25:21,837 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41487 is added to blk_1073741939_1119 (size=93) 2024-12-03T23:25:21,837 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45491 is added to blk_1073741939_1119 (size=93) 2024-12-03T23:25:21,837 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39829 is added to blk_1073741939_1119 (size=93) 2024-12-03T23:25:21,839 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-03T23:25:21,839 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1733268321682) 2024-12-03T23:25:21,854 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayCompressed#testReplayEditsWrittenViaHRegion Thread=450 (was 442) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1462297639_22 at /127.0.0.1:34698 [Waiting for operation #8] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-28-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1462297639_22 at /127.0.0.1:47064 [Waiting for operation #23] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkinstestReplayEditsWrittenViaHRegion@localhost:35563 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-28-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1462297639_22 at /127.0.0.1:41436 [Waiting for operation #12] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1987708253) connection to localhost/127.0.0.1:35563 from jenkinstestReplayEditsWrittenViaHRegion java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: AsyncFSWAL-28-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1414 (was 1346) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=192 (was 209), ProcessCount=11 (was 11), AvailableMemoryMB=7767 (was 7778) 2024-12-03T23:25:21,855 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1414 is superior to 1024 2024-12-03T23:25:21,855 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-12-03T23:25:21,855 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-03T23:25:21,855 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-03T23:25:21,855 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T23:25:21,856 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T23:25:21,856 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T23:25:21,856 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-12-03T23:25:21,856 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=315704162, stopped=false 2024-12-03T23:25:21,856 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=5f9d64419733,41995,1733268282840 2024-12-03T23:25:21,858 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44215-0x1017aa5055b0002, quorum=127.0.0.1:51247, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-03T23:25:21,858 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41995-0x1017aa5055b0000, quorum=127.0.0.1:51247, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-03T23:25:21,858 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44411-0x1017aa5055b0003, quorum=127.0.0.1:51247, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-03T23:25:21,858 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44215-0x1017aa5055b0002, quorum=127.0.0.1:51247, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T23:25:21,858 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44411-0x1017aa5055b0003, quorum=127.0.0.1:51247, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T23:25:21,858 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-03T23:25:21,858 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41995-0x1017aa5055b0000, quorum=127.0.0.1:51247, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T23:25:21,858 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-03T23:25:21,859 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-03T23:25:21,859 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T23:25:21,859 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:44411-0x1017aa5055b0003, quorum=127.0.0.1:51247, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-03T23:25:21,859 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '5f9d64419733,44215,1733268283642' ***** 2024-12-03T23:25:21,859 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-03T23:25:21,859 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '5f9d64419733,44411,1733268283680' ***** 2024-12-03T23:25:21,859 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-03T23:25:21,859 INFO [RS:1;5f9d64419733:44215 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-03T23:25:21,859 INFO [RS:2;5f9d64419733:44411 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-03T23:25:21,859 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:44215-0x1017aa5055b0002, quorum=127.0.0.1:51247, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-03T23:25:21,860 INFO [RS:1;5f9d64419733:44215 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-03T23:25:21,860 INFO [RS:2;5f9d64419733:44411 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-03T23:25:21,860 INFO [RS:1;5f9d64419733:44215 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-03T23:25:21,860 INFO [RS:2;5f9d64419733:44411 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-03T23:25:21,860 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:41995-0x1017aa5055b0000, quorum=127.0.0.1:51247, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-03T23:25:21,860 INFO [RS:1;5f9d64419733:44215 {}] regionserver.HRegionServer(959): stopping server 5f9d64419733,44215,1733268283642 2024-12-03T23:25:21,860 INFO [RS:2;5f9d64419733:44411 {}] regionserver.HRegionServer(3091): Received CLOSE for 1745e7d6d10a6c6039d4c33aadf95a2f 2024-12-03T23:25:21,860 INFO [RS:1;5f9d64419733:44215 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-03T23:25:21,860 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-03T23:25:21,860 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-03T23:25:21,860 INFO [RS:1;5f9d64419733:44215 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;5f9d64419733:44215. 2024-12-03T23:25:21,860 DEBUG [RS:1;5f9d64419733:44215 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-03T23:25:21,860 DEBUG [RS:1;5f9d64419733:44215 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T23:25:21,860 INFO [RS:1;5f9d64419733:44215 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-03T23:25:21,860 INFO [RS:1;5f9d64419733:44215 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-03T23:25:21,860 INFO [RS:1;5f9d64419733:44215 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-03T23:25:21,860 INFO [RS:1;5f9d64419733:44215 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-12-03T23:25:21,860 INFO [RS:2;5f9d64419733:44411 {}] regionserver.HRegionServer(959): stopping server 5f9d64419733,44411,1733268283680 2024-12-03T23:25:21,860 INFO [RS:2;5f9d64419733:44411 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-03T23:25:21,861 INFO [RS:2;5f9d64419733:44411 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:2;5f9d64419733:44411. 2024-12-03T23:25:21,861 DEBUG [RS:2;5f9d64419733:44411 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-03T23:25:21,861 INFO [RS:1;5f9d64419733:44215 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-12-03T23:25:21,861 DEBUG [RS:2;5f9d64419733:44411 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T23:25:21,861 DEBUG [RS:1;5f9d64419733:44215 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-12-03T23:25:21,861 DEBUG [RS:1;5f9d64419733:44215 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-03T23:25:21,861 INFO [RS:2;5f9d64419733:44411 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-12-03T23:25:21,861 DEBUG [RS:2;5f9d64419733:44411 {}] regionserver.HRegionServer(1325): Online Regions={1745e7d6d10a6c6039d4c33aadf95a2f=testReplayEditsAfterRegionMovedWithMultiCF,,1733268298272.1745e7d6d10a6c6039d4c33aadf95a2f.} 2024-12-03T23:25:21,861 DEBUG [RS:2;5f9d64419733:44411 {}] regionserver.HRegionServer(1351): Waiting on 1745e7d6d10a6c6039d4c33aadf95a2f 2024-12-03T23:25:21,861 DEBUG [RS_CLOSE_META-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-03T23:25:21,861 INFO [RS_CLOSE_META-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-03T23:25:21,861 DEBUG [RS_CLOSE_META-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-03T23:25:21,861 DEBUG [RS_CLOSE_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 1745e7d6d10a6c6039d4c33aadf95a2f, disabling compactions & flushes 2024-12-03T23:25:21,861 INFO [RS_CLOSE_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1733268298272.1745e7d6d10a6c6039d4c33aadf95a2f. 2024-12-03T23:25:21,861 DEBUG [RS_CLOSE_META-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-03T23:25:21,861 DEBUG [RS_CLOSE_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733268298272.1745e7d6d10a6c6039d4c33aadf95a2f. 2024-12-03T23:25:21,861 DEBUG [RS_CLOSE_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733268298272.1745e7d6d10a6c6039d4c33aadf95a2f. after waiting 0 ms 2024-12-03T23:25:21,861 DEBUG [RS_CLOSE_META-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-03T23:25:21,861 DEBUG [RS_CLOSE_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1733268298272.1745e7d6d10a6c6039d4c33aadf95a2f. 2024-12-03T23:25:21,861 INFO [RS_CLOSE_META-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.19 KB heapSize=2.79 KB 2024-12-03T23:25:21,864 INFO [regionserver/5f9d64419733:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-03T23:25:21,866 DEBUG [RS_CLOSE_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35563/user/jenkins/test-data/e9061e95-7d9e-c12a-b652-9e8a1d87f0cf/data/default/testReplayEditsAfterRegionMovedWithMultiCF/1745e7d6d10a6c6039d4c33aadf95a2f/recovered.edits/20.seqid, newMaxSeqId=20, maxSeqId=17 2024-12-03T23:25:21,867 INFO [RS_CLOSE_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733268298272.1745e7d6d10a6c6039d4c33aadf95a2f. 2024-12-03T23:25:21,867 DEBUG [RS_CLOSE_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 1745e7d6d10a6c6039d4c33aadf95a2f: Waiting for close lock at 1733268321861Running coprocessor pre-close hooks at 1733268321861Disabling compacts and flushes for region at 1733268321861Disabling writes for close at 1733268321861Writing region close event to WAL at 1733268321862 (+1 ms)Running coprocessor post-close hooks at 1733268321867 (+5 ms)Closed at 1733268321867 2024-12-03T23:25:21,867 DEBUG [RS_CLOSE_REGION-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733268298272.1745e7d6d10a6c6039d4c33aadf95a2f. 2024-12-03T23:25:21,867 DEBUG [RS_CLOSE_META-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35563/user/jenkins/test-data/e9061e95-7d9e-c12a-b652-9e8a1d87f0cf/data/hbase/meta/1588230740/.tmp/info/ca7b7bba082640f8aaa575495a40c289 is 205, key is testReplayEditsAfterRegionMovedWithMultiCF,,1733268298272.1745e7d6d10a6c6039d4c33aadf95a2f./info:regioninfo/1733268302151/Put/seqid=0 2024-12-03T23:25:21,867 INFO [regionserver/5f9d64419733:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-03T23:25:21,874 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39829 is added to blk_1073741943_1123 (size=6778) 2024-12-03T23:25:21,874 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41487 is added to blk_1073741943_1123 (size=6778) 2024-12-03T23:25:21,875 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45491 is added to blk_1073741943_1123 (size=6778) 2024-12-03T23:25:21,875 INFO [RS_CLOSE_META-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.19 KB at sequenceid=23 (bloomFilter=true), to=hdfs://localhost:35563/user/jenkins/test-data/e9061e95-7d9e-c12a-b652-9e8a1d87f0cf/data/hbase/meta/1588230740/.tmp/info/ca7b7bba082640f8aaa575495a40c289 2024-12-03T23:25:21,881 DEBUG [RS_CLOSE_META-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35563/user/jenkins/test-data/e9061e95-7d9e-c12a-b652-9e8a1d87f0cf/data/hbase/meta/1588230740/.tmp/info/ca7b7bba082640f8aaa575495a40c289 as hdfs://localhost:35563/user/jenkins/test-data/e9061e95-7d9e-c12a-b652-9e8a1d87f0cf/data/hbase/meta/1588230740/info/ca7b7bba082640f8aaa575495a40c289 2024-12-03T23:25:21,886 INFO [RS_CLOSE_META-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35563/user/jenkins/test-data/e9061e95-7d9e-c12a-b652-9e8a1d87f0cf/data/hbase/meta/1588230740/info/ca7b7bba082640f8aaa575495a40c289, entries=8, sequenceid=23, filesize=6.6 K 2024-12-03T23:25:21,888 INFO [RS_CLOSE_META-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~1.19 KB/1218, heapSize ~2.02 KB/2072, currentSize=0 B/0 for 1588230740 in 26ms, sequenceid=23, compaction requested=false 2024-12-03T23:25:21,888 DEBUG [RS_CLOSE_META-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_META}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-12-03T23:25:21,892 DEBUG [RS_CLOSE_META-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35563/user/jenkins/test-data/e9061e95-7d9e-c12a-b652-9e8a1d87f0cf/data/hbase/meta/1588230740/recovered.edits/26.seqid, newMaxSeqId=26, maxSeqId=18 2024-12-03T23:25:21,893 DEBUG [RS_CLOSE_META-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-03T23:25:21,893 INFO [RS_CLOSE_META-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-03T23:25:21,893 DEBUG [RS_CLOSE_META-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733268321861Running coprocessor pre-close hooks at 1733268321861Disabling compacts and flushes for region at 1733268321861Disabling writes for close at 1733268321861Obtaining lock to block concurrent updates at 1733268321861Preparing flush snapshotting stores in 1588230740 at 1733268321861Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=1218, getHeapSize=2792, getOffHeapSize=0, getCellsCount=8 at 1733268321862 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1733268321863 (+1 ms)Flushing 1588230740/info: creating writer at 1733268321863Flushing 1588230740/info: appending metadata at 1733268321867 (+4 ms)Flushing 1588230740/info: closing flushed file at 1733268321867Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5c729953: reopening flushed file at 1733268321881 (+14 ms)Finished flush of dataSize ~1.19 KB/1218, heapSize ~2.02 KB/2072, currentSize=0 B/0 for 1588230740 in 26ms, sequenceid=23, compaction requested=false at 1733268321888 (+7 ms)Writing region close event to WAL at 1733268321889 (+1 ms)Running coprocessor post-close hooks at 1733268321893 (+4 ms)Closed at 1733268321893 2024-12-03T23:25:21,893 DEBUG [RS_CLOSE_META-regionserver/5f9d64419733:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-12-03T23:25:21,911 INFO [regionserver/5f9d64419733:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-12-03T23:25:21,911 INFO [regionserver/5f9d64419733:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-12-03T23:25:22,061 INFO [RS:1;5f9d64419733:44215 {}] regionserver.HRegionServer(976): stopping server 5f9d64419733,44215,1733268283642; all regions closed. 2024-12-03T23:25:22,061 INFO [RS:2;5f9d64419733:44411 {}] regionserver.HRegionServer(976): stopping server 5f9d64419733,44411,1733268283680; all regions closed. 2024-12-03T23:25:22,064 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41487 is added to blk_1073741833_1009 (size=841) 2024-12-03T23:25:22,065 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39829 is added to blk_1073741893_1071 (size=1552) 2024-12-03T23:25:22,065 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39829 is added to blk_1073741833_1009 (size=841) 2024-12-03T23:25:22,065 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45491 is added to blk_1073741893_1071 (size=1552) 2024-12-03T23:25:22,065 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41487 is added to blk_1073741893_1071 (size=1552) 2024-12-03T23:25:22,065 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45491 is added to blk_1073741833_1009 (size=841) 2024-12-03T23:25:22,067 DEBUG [RS:1;5f9d64419733:44215 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/e9061e95-7d9e-c12a-b652-9e8a1d87f0cf/oldWALs 2024-12-03T23:25:22,067 INFO [RS:1;5f9d64419733:44215 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 5f9d64419733%2C44215%2C1733268283642.meta:.meta(num 1733268301314) 2024-12-03T23:25:22,070 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45491 is added to blk_1073741835_1011 (size=2054) 2024-12-03T23:25:22,070 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41487 is added to blk_1073741835_1011 (size=2054) 2024-12-03T23:25:22,070 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39829 is added to blk_1073741835_1011 (size=2054) 2024-12-03T23:25:22,070 DEBUG [RS:2;5f9d64419733:44411 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/e9061e95-7d9e-c12a-b652-9e8a1d87f0cf/oldWALs 2024-12-03T23:25:22,070 INFO [RS:2;5f9d64419733:44411 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 5f9d64419733%2C44411%2C1733268283680:(num 1733268285003) 2024-12-03T23:25:22,070 DEBUG [RS:2;5f9d64419733:44411 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T23:25:22,070 INFO [RS:2;5f9d64419733:44411 {}] regionserver.LeaseManager(133): Closed leases 2024-12-03T23:25:22,071 INFO [RS:2;5f9d64419733:44411 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-03T23:25:22,071 INFO [RS:2;5f9d64419733:44411 {}] hbase.ChoreService(370): Chore service for: regionserver/5f9d64419733:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-12-03T23:25:22,071 INFO [RS:2;5f9d64419733:44411 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-03T23:25:22,071 INFO [RS:2;5f9d64419733:44411 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-03T23:25:22,071 INFO [RS:2;5f9d64419733:44411 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-03T23:25:22,071 INFO [regionserver/5f9d64419733:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-03T23:25:22,071 INFO [RS:2;5f9d64419733:44411 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-03T23:25:22,071 INFO [RS:2;5f9d64419733:44411 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:44411 2024-12-03T23:25:22,073 DEBUG [RS:1;5f9d64419733:44215 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/e9061e95-7d9e-c12a-b652-9e8a1d87f0cf/oldWALs 2024-12-03T23:25:22,073 INFO [RS:1;5f9d64419733:44215 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 5f9d64419733%2C44215%2C1733268283642:(num 1733268285003) 2024-12-03T23:25:22,073 DEBUG [RS:1;5f9d64419733:44215 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T23:25:22,073 INFO [RS:1;5f9d64419733:44215 {}] regionserver.LeaseManager(133): Closed leases 2024-12-03T23:25:22,073 INFO [RS:1;5f9d64419733:44215 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-03T23:25:22,073 INFO [RS:1;5f9d64419733:44215 {}] hbase.ChoreService(370): Chore service for: regionserver/5f9d64419733:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-12-03T23:25:22,073 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44411-0x1017aa5055b0003, quorum=127.0.0.1:51247, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/5f9d64419733,44411,1733268283680 2024-12-03T23:25:22,073 INFO [RS:1;5f9d64419733:44215 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-03T23:25:22,073 INFO [regionserver/5f9d64419733:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-03T23:25:22,073 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41995-0x1017aa5055b0000, quorum=127.0.0.1:51247, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-03T23:25:22,073 INFO [RS:1;5f9d64419733:44215 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:44215 2024-12-03T23:25:22,074 INFO [RS:2;5f9d64419733:44411 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-03T23:25:22,075 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [5f9d64419733,44411,1733268283680] 2024-12-03T23:25:22,078 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44215-0x1017aa5055b0002, quorum=127.0.0.1:51247, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/5f9d64419733,44215,1733268283642 2024-12-03T23:25:22,078 INFO [RS:1;5f9d64419733:44215 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-03T23:25:22,078 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41995-0x1017aa5055b0000, quorum=127.0.0.1:51247, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-03T23:25:22,078 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/5f9d64419733,44411,1733268283680 already deleted, retry=false 2024-12-03T23:25:22,078 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 5f9d64419733,44411,1733268283680 expired; onlineServers=1 2024-12-03T23:25:22,079 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [5f9d64419733,44215,1733268283642] 2024-12-03T23:25:22,080 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/5f9d64419733,44215,1733268283642 already deleted, retry=false 2024-12-03T23:25:22,081 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 5f9d64419733,44215,1733268283642 expired; onlineServers=0 2024-12-03T23:25:22,081 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '5f9d64419733,41995,1733268282840' ***** 2024-12-03T23:25:22,081 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-12-03T23:25:22,081 INFO [M:0;5f9d64419733:41995 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-03T23:25:22,081 INFO [M:0;5f9d64419733:41995 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-03T23:25:22,081 DEBUG [M:0;5f9d64419733:41995 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-12-03T23:25:22,081 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-12-03T23:25:22,081 DEBUG [M:0;5f9d64419733:41995 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-12-03T23:25:22,081 DEBUG [master/5f9d64419733:0:becomeActiveMaster-HFileCleaner.large.0-1733268284664 {}] cleaner.HFileCleaner(306): Exit Thread[master/5f9d64419733:0:becomeActiveMaster-HFileCleaner.large.0-1733268284664,5,FailOnTimeoutGroup] 2024-12-03T23:25:22,081 DEBUG [master/5f9d64419733:0:becomeActiveMaster-HFileCleaner.small.0-1733268284667 {}] cleaner.HFileCleaner(306): Exit Thread[master/5f9d64419733:0:becomeActiveMaster-HFileCleaner.small.0-1733268284667,5,FailOnTimeoutGroup] 2024-12-03T23:25:22,081 INFO [M:0;5f9d64419733:41995 {}] hbase.ChoreService(370): Chore service for: master/5f9d64419733:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-12-03T23:25:22,082 INFO [M:0;5f9d64419733:41995 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-03T23:25:22,082 DEBUG [M:0;5f9d64419733:41995 {}] master.HMaster(1795): Stopping service threads 2024-12-03T23:25:22,082 INFO [M:0;5f9d64419733:41995 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-12-03T23:25:22,082 INFO [M:0;5f9d64419733:41995 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-03T23:25:22,082 INFO [M:0;5f9d64419733:41995 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-03T23:25:22,083 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-12-03T23:25:22,084 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41995-0x1017aa5055b0000, quorum=127.0.0.1:51247, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-12-03T23:25:22,084 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41995-0x1017aa5055b0000, quorum=127.0.0.1:51247, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T23:25:22,084 DEBUG [M:0;5f9d64419733:41995 {}] zookeeper.ZKUtil(347): master:41995-0x1017aa5055b0000, quorum=127.0.0.1:51247, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-12-03T23:25:22,085 WARN [M:0;5f9d64419733:41995 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-12-03T23:25:22,085 INFO [M:0;5f9d64419733:41995 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:35563/user/jenkins/test-data/e9061e95-7d9e-c12a-b652-9e8a1d87f0cf/.lastflushedseqids 2024-12-03T23:25:22,097 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39829 is added to blk_1073741944_1124 (size=119) 2024-12-03T23:25:22,097 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41487 is added to blk_1073741944_1124 (size=119) 2024-12-03T23:25:22,098 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45491 is added to blk_1073741944_1124 (size=119) 2024-12-03T23:25:22,098 INFO [M:0;5f9d64419733:41995 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-03T23:25:22,098 INFO [M:0;5f9d64419733:41995 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-12-03T23:25:22,099 DEBUG [M:0;5f9d64419733:41995 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-03T23:25:22,099 INFO [M:0;5f9d64419733:41995 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-03T23:25:22,099 DEBUG [M:0;5f9d64419733:41995 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-03T23:25:22,099 DEBUG [M:0;5f9d64419733:41995 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-03T23:25:22,099 DEBUG [M:0;5f9d64419733:41995 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-03T23:25:22,099 INFO [M:0;5f9d64419733:41995 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=83.41 KB heapSize=102.71 KB 2024-12-03T23:25:22,114 DEBUG [M:0;5f9d64419733:41995 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35563/user/jenkins/test-data/e9061e95-7d9e-c12a-b652-9e8a1d87f0cf/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/aa496b2b0ff64d27bb20cf396b3492a7 is 82, key is hbase:meta,,1/info:regioninfo/1733268301485/Put/seqid=0 2024-12-03T23:25:22,120 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39829 is added to blk_1073741945_1125 (size=6063) 2024-12-03T23:25:22,120 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41487 is added to blk_1073741945_1125 (size=6063) 2024-12-03T23:25:22,120 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45491 is added to blk_1073741945_1125 (size=6063) 2024-12-03T23:25:22,122 INFO [M:0;5f9d64419733:41995 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1008 B at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:35563/user/jenkins/test-data/e9061e95-7d9e-c12a-b652-9e8a1d87f0cf/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/aa496b2b0ff64d27bb20cf396b3492a7 2024-12-03T23:25:22,149 DEBUG [M:0;5f9d64419733:41995 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35563/user/jenkins/test-data/e9061e95-7d9e-c12a-b652-9e8a1d87f0cf/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/213aa56e2b8544e48abd211401058d7b is 1077, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1733268298693/Put/seqid=0 2024-12-03T23:25:22,157 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41487 is added to blk_1073741946_1126 (size=7908) 2024-12-03T23:25:22,157 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39829 is added to blk_1073741946_1126 (size=7908) 2024-12-03T23:25:22,157 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45491 is added to blk_1073741946_1126 (size=7908) 2024-12-03T23:25:22,158 INFO [M:0;5f9d64419733:41995 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=82.18 KB at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:35563/user/jenkins/test-data/e9061e95-7d9e-c12a-b652-9e8a1d87f0cf/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/213aa56e2b8544e48abd211401058d7b 2024-12-03T23:25:22,163 INFO [M:0;5f9d64419733:41995 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 213aa56e2b8544e48abd211401058d7b 2024-12-03T23:25:22,177 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44411-0x1017aa5055b0003, quorum=127.0.0.1:51247, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-03T23:25:22,177 INFO [RS:2;5f9d64419733:44411 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-03T23:25:22,177 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44411-0x1017aa5055b0003, quorum=127.0.0.1:51247, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-03T23:25:22,177 INFO [RS:2;5f9d64419733:44411 {}] regionserver.HRegionServer(1031): Exiting; stopping=5f9d64419733,44411,1733268283680; zookeeper connection closed. 2024-12-03T23:25:22,178 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@6909df3d {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@6909df3d 2024-12-03T23:25:22,178 DEBUG [M:0;5f9d64419733:41995 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35563/user/jenkins/test-data/e9061e95-7d9e-c12a-b652-9e8a1d87f0cf/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/2d5bffbe5b364aec9c4ddca2f9bb33fa is 69, key is 5f9d64419733,44215,1733268283642/rs:state/1733268284751/Put/seqid=0 2024-12-03T23:25:22,179 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44215-0x1017aa5055b0002, quorum=127.0.0.1:51247, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-03T23:25:22,179 INFO [RS:1;5f9d64419733:44215 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-03T23:25:22,179 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44215-0x1017aa5055b0002, quorum=127.0.0.1:51247, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-03T23:25:22,180 INFO [RS:1;5f9d64419733:44215 {}] regionserver.HRegionServer(1031): Exiting; stopping=5f9d64419733,44215,1733268283642; zookeeper connection closed. 2024-12-03T23:25:22,180 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@39e4d455 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@39e4d455 2024-12-03T23:25:22,180 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 3 regionserver(s) complete 2024-12-03T23:25:22,184 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41487 is added to blk_1073741947_1127 (size=5440) 2024-12-03T23:25:22,184 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39829 is added to blk_1073741947_1127 (size=5440) 2024-12-03T23:25:22,184 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45491 is added to blk_1073741947_1127 (size=5440) 2024-12-03T23:25:22,185 INFO [M:0;5f9d64419733:41995 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=249 B at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:35563/user/jenkins/test-data/e9061e95-7d9e-c12a-b652-9e8a1d87f0cf/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/2d5bffbe5b364aec9c4ddca2f9bb33fa 2024-12-03T23:25:22,190 INFO [M:0;5f9d64419733:41995 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 2d5bffbe5b364aec9c4ddca2f9bb33fa 2024-12-03T23:25:22,191 DEBUG [M:0;5f9d64419733:41995 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35563/user/jenkins/test-data/e9061e95-7d9e-c12a-b652-9e8a1d87f0cf/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/aa496b2b0ff64d27bb20cf396b3492a7 as hdfs://localhost:35563/user/jenkins/test-data/e9061e95-7d9e-c12a-b652-9e8a1d87f0cf/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/aa496b2b0ff64d27bb20cf396b3492a7 2024-12-03T23:25:22,195 INFO [M:0;5f9d64419733:41995 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35563/user/jenkins/test-data/e9061e95-7d9e-c12a-b652-9e8a1d87f0cf/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/aa496b2b0ff64d27bb20cf396b3492a7, entries=14, sequenceid=207, filesize=5.9 K 2024-12-03T23:25:22,196 DEBUG [M:0;5f9d64419733:41995 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35563/user/jenkins/test-data/e9061e95-7d9e-c12a-b652-9e8a1d87f0cf/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/213aa56e2b8544e48abd211401058d7b as hdfs://localhost:35563/user/jenkins/test-data/e9061e95-7d9e-c12a-b652-9e8a1d87f0cf/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/213aa56e2b8544e48abd211401058d7b 2024-12-03T23:25:22,201 INFO [M:0;5f9d64419733:41995 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 213aa56e2b8544e48abd211401058d7b 2024-12-03T23:25:22,201 INFO [M:0;5f9d64419733:41995 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35563/user/jenkins/test-data/e9061e95-7d9e-c12a-b652-9e8a1d87f0cf/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/213aa56e2b8544e48abd211401058d7b, entries=21, sequenceid=207, filesize=7.7 K 2024-12-03T23:25:22,202 DEBUG [M:0;5f9d64419733:41995 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35563/user/jenkins/test-data/e9061e95-7d9e-c12a-b652-9e8a1d87f0cf/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/2d5bffbe5b364aec9c4ddca2f9bb33fa as hdfs://localhost:35563/user/jenkins/test-data/e9061e95-7d9e-c12a-b652-9e8a1d87f0cf/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/2d5bffbe5b364aec9c4ddca2f9bb33fa 2024-12-03T23:25:22,206 INFO [M:0;5f9d64419733:41995 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 2d5bffbe5b364aec9c4ddca2f9bb33fa 2024-12-03T23:25:22,206 INFO [M:0;5f9d64419733:41995 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35563/user/jenkins/test-data/e9061e95-7d9e-c12a-b652-9e8a1d87f0cf/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/2d5bffbe5b364aec9c4ddca2f9bb33fa, entries=3, sequenceid=207, filesize=5.3 K 2024-12-03T23:25:22,207 INFO [M:0;5f9d64419733:41995 {}] regionserver.HRegion(3140): Finished flush of dataSize ~83.41 KB/85410, heapSize ~102.41 KB/104872, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 108ms, sequenceid=207, compaction requested=false 2024-12-03T23:25:22,209 INFO [M:0;5f9d64419733:41995 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-03T23:25:22,209 DEBUG [M:0;5f9d64419733:41995 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733268322098Disabling compacts and flushes for region at 1733268322098Disabling writes for close at 1733268322099 (+1 ms)Obtaining lock to block concurrent updates at 1733268322099Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1733268322099Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=85410, getHeapSize=105112, getOffHeapSize=0, getCellsCount=248 at 1733268322099Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1733268322100 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1733268322100Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1733268322114 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1733268322114Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1733268322127 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1733268322148 (+21 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1733268322148Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1733268322163 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1733268322177 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1733268322177Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@480822c: reopening flushed file at 1733268322190 (+13 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@77491d41: reopening flushed file at 1733268322195 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@14a524ea: reopening flushed file at 1733268322201 (+6 ms)Finished flush of dataSize ~83.41 KB/85410, heapSize ~102.41 KB/104872, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 108ms, sequenceid=207, compaction requested=false at 1733268322207 (+6 ms)Writing region close event to WAL at 1733268322209 (+2 ms)Closed at 1733268322209 2024-12-03T23:25:22,211 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45491 is added to blk_1073741830_1006 (size=86759) 2024-12-03T23:25:22,211 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39829 is added to blk_1073741830_1006 (size=86759) 2024-12-03T23:25:22,212 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41487 is added to blk_1073741830_1006 (size=86759) 2024-12-03T23:25:22,212 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-03T23:25:22,213 INFO [M:0;5f9d64419733:41995 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-12-03T23:25:22,213 INFO [M:0;5f9d64419733:41995 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:41995 2024-12-03T23:25:22,213 INFO [M:0;5f9d64419733:41995 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-03T23:25:22,315 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41995-0x1017aa5055b0000, quorum=127.0.0.1:51247, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-03T23:25:22,315 INFO [M:0;5f9d64419733:41995 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-03T23:25:22,315 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41995-0x1017aa5055b0000, quorum=127.0.0.1:51247, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-03T23:25:22,322 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733268302792/wal.1733268312038 with renewLeaseKey: DEFAULT_16688 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733268302792/wal.1733268312038 (inode 16688) Holder DFSClient_NONMAPREDUCE_-773788190_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733268302792/wal.1733268312038 (inode 16688) Holder DFSClient_NONMAPREDUCE_-773788190_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy45.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy46.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-12-03T23:25:22,325 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733268302372/wal.1733268302658 with renewLeaseKey: DEFAULT_16665 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733268302372/wal.1733268302658 (inode 16665) Holder DFSClient_NONMAPREDUCE_-773788190_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733268302372/wal.1733268302658 (inode 16665) Holder DFSClient_NONMAPREDUCE_-773788190_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy45.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy46.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-12-03T23:25:22,325 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733268293902/wal.1733268293990 with renewLeaseKey: DEFAULT_16586 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T23:25:22,327 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testdatalosswheninputerror-manual,16010,1733268312277/wal.1733268312756 with renewLeaseKey: DEFAULT_16714 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testdatalosswheninputerror-manual,16010,1733268312277/wal.1733268312756 (inode 16714) Holder DFSClient_NONMAPREDUCE_-773788190_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testdatalosswheninputerror-manual,16010,1733268312277/wal.1733268312756 (inode 16714) Holder DFSClient_NONMAPREDUCE_-773788190_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy45.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy46.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-12-03T23:25:22,328 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733268312894/wal.1733268312949 with renewLeaseKey: DEFAULT_16736 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T23:25:22,328 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733268317293/wal.1733268317486 with renewLeaseKey: DEFAULT_16777 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T23:25:22,328 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733268286722/wal.1733268286795 with renewLeaseKey: DEFAULT_16506 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T23:25:22,330 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testnameconflictwhensplit1-manual,16010,1733268286402/wal.1733268286588 with renewLeaseKey: DEFAULT_16485 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testnameconflictwhensplit1-manual,16010,1733268286402/wal.1733268286588 (inode 16485) Holder DFSClient_NONMAPREDUCE_-773788190_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testnameconflictwhensplit1-manual,16010,1733268286402/wal.1733268286588 (inode 16485) Holder DFSClient_NONMAPREDUCE_-773788190_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy45.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy46.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-12-03T23:25:22,332 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testnameconflictwhensplit0-manual,16010,1733268285908/wal.1733268286142 with renewLeaseKey: DEFAULT_16462 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testnameconflictwhensplit0-manual,16010,1733268285908/wal.1733268286142 (inode 16462) Holder DFSClient_NONMAPREDUCE_-773788190_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testnameconflictwhensplit0-manual,16010,1733268285908/wal.1733268286142 (inode 16462) Holder DFSClient_NONMAPREDUCE_-773788190_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy45.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy46.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-12-03T23:25:22,337 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4290616c{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-03T23:25:22,339 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@5e25d2b6{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-03T23:25:22,339 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-03T23:25:22,339 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6853f5bc{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-03T23:25:22,339 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1bf0fdb4{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/246ab1eb-e73c-fcd7-a82a-bcf4c34f0add/hadoop.log.dir/,STOPPED} 2024-12-03T23:25:22,342 WARN [BP-2057003906-172.17.0.2-1733268279817 heartbeating to localhost/127.0.0.1:35563 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-03T23:25:22,342 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-03T23:25:22,342 WARN [BP-2057003906-172.17.0.2-1733268279817 heartbeating to localhost/127.0.0.1:35563 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-2057003906-172.17.0.2-1733268279817 (Datanode Uuid 30e7d33a-aa1d-4010-b6b0-785ae5df5297) service to localhost/127.0.0.1:35563 2024-12-03T23:25:22,342 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-03T23:25:22,343 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/246ab1eb-e73c-fcd7-a82a-bcf4c34f0add/cluster_eb757c39-dfd9-3b39-78fd-eea66f4cff1d/data/data5/current/BP-2057003906-172.17.0.2-1733268279817 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-03T23:25:22,343 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/246ab1eb-e73c-fcd7-a82a-bcf4c34f0add/cluster_eb757c39-dfd9-3b39-78fd-eea66f4cff1d/data/data6/current/BP-2057003906-172.17.0.2-1733268279817 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-03T23:25:22,344 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-03T23:25:22,346 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@28e1ba78{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-03T23:25:22,346 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@702c0733{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-03T23:25:22,346 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-03T23:25:22,346 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3264f7d{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-03T23:25:22,346 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@e971547{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/246ab1eb-e73c-fcd7-a82a-bcf4c34f0add/hadoop.log.dir/,STOPPED} 2024-12-03T23:25:22,349 WARN [BP-2057003906-172.17.0.2-1733268279817 heartbeating to localhost/127.0.0.1:35563 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-03T23:25:22,349 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-03T23:25:22,349 WARN [BP-2057003906-172.17.0.2-1733268279817 heartbeating to localhost/127.0.0.1:35563 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-2057003906-172.17.0.2-1733268279817 (Datanode Uuid d14c7b45-d11a-4744-a24a-fef0e3de8324) service to localhost/127.0.0.1:35563 2024-12-03T23:25:22,349 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-03T23:25:22,349 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/246ab1eb-e73c-fcd7-a82a-bcf4c34f0add/cluster_eb757c39-dfd9-3b39-78fd-eea66f4cff1d/data/data3/current/BP-2057003906-172.17.0.2-1733268279817 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-03T23:25:22,350 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/246ab1eb-e73c-fcd7-a82a-bcf4c34f0add/cluster_eb757c39-dfd9-3b39-78fd-eea66f4cff1d/data/data4/current/BP-2057003906-172.17.0.2-1733268279817 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-03T23:25:22,350 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-03T23:25:22,352 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4dc44ec6{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-03T23:25:22,352 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@17f6e478{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-03T23:25:22,352 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-03T23:25:22,352 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3d33512e{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-03T23:25:22,352 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4b895d95{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/246ab1eb-e73c-fcd7-a82a-bcf4c34f0add/hadoop.log.dir/,STOPPED} 2024-12-03T23:25:22,353 WARN [BP-2057003906-172.17.0.2-1733268279817 heartbeating to localhost/127.0.0.1:35563 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-03T23:25:22,353 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-03T23:25:22,353 WARN [BP-2057003906-172.17.0.2-1733268279817 heartbeating to localhost/127.0.0.1:35563 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-2057003906-172.17.0.2-1733268279817 (Datanode Uuid 31ac79b2-eca3-47f4-bfac-d43617ab8328) service to localhost/127.0.0.1:35563 2024-12-03T23:25:22,353 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-03T23:25:22,354 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/246ab1eb-e73c-fcd7-a82a-bcf4c34f0add/cluster_eb757c39-dfd9-3b39-78fd-eea66f4cff1d/data/data1/current/BP-2057003906-172.17.0.2-1733268279817 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-03T23:25:22,354 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-03T23:25:22,354 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/246ab1eb-e73c-fcd7-a82a-bcf4c34f0add/cluster_eb757c39-dfd9-3b39-78fd-eea66f4cff1d/data/data2/current/BP-2057003906-172.17.0.2-1733268279817 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-03T23:25:22,360 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@1843526c{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-03T23:25:22,361 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@776c4d96{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-03T23:25:22,361 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-03T23:25:22,361 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@70596a48{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-03T23:25:22,361 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6630be9c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/246ab1eb-e73c-fcd7-a82a-bcf4c34f0add/hadoop.log.dir/,STOPPED} 2024-12-03T23:25:22,371 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-12-03T23:25:22,419 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down