2024-11-16 14:32:06,632 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@20b2475a 2024-11-16 14:32:06,644 main DEBUG Took 0.009603 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-16 14:32:06,645 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-16 14:32:06,645 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-16 14:32:06,646 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-16 14:32:06,647 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-16 14:32:06,654 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-16 14:32:06,666 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 14:32:06,667 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-16 14:32:06,668 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 14:32:06,668 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-16 14:32:06,669 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 14:32:06,669 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-16 14:32:06,670 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 14:32:06,670 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-16 14:32:06,671 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 14:32:06,671 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-16 14:32:06,672 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 14:32:06,673 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-16 14:32:06,673 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 14:32:06,673 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-16 14:32:06,674 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 14:32:06,674 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-16 14:32:06,675 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 14:32:06,675 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-16 14:32:06,675 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 14:32:06,676 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-16 14:32:06,676 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 14:32:06,676 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-16 14:32:06,677 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 14:32:06,677 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-16 14:32:06,678 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 14:32:06,678 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-16 14:32:06,679 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 14:32:06,681 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-16 14:32:06,682 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-16 14:32:06,683 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-16 14:32:06,684 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-16 14:32:06,684 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-16 14:32:06,692 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-16 14:32:06,695 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-16 14:32:06,697 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-16 14:32:06,697 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-16 14:32:06,697 main DEBUG createAppenders(={Console}) 2024-11-16 14:32:06,698 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@20b2475a initialized 2024-11-16 14:32:06,698 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@20b2475a 2024-11-16 14:32:06,699 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@20b2475a OK. 2024-11-16 14:32:06,699 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-16 14:32:06,699 main DEBUG OutputStream closed 2024-11-16 14:32:06,700 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-16 14:32:06,700 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-16 14:32:06,700 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@4310d43 OK 2024-11-16 14:32:06,771 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-16 14:32:06,773 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-16 14:32:06,775 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-16 14:32:06,776 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-16 14:32:06,776 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-16 14:32:06,777 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-16 14:32:06,777 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-16 14:32:06,777 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-16 14:32:06,778 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-16 14:32:06,778 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-16 14:32:06,778 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-16 14:32:06,779 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-16 14:32:06,779 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-16 14:32:06,779 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-16 14:32:06,780 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-16 14:32:06,780 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-16 14:32:06,780 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-16 14:32:06,781 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-16 14:32:06,783 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-16 14:32:06,784 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@12f9af83) with optional ClassLoader: null 2024-11-16 14:32:06,784 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-16 14:32:06,785 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@12f9af83] started OK. 2024-11-16T14:32:07,014 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/650842a5-0ab8-4683-35e8-38af23b31d67 2024-11-16 14:32:07,017 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-16 14:32:07,017 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-16T14:32:07,026 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay timeout: 13 mins 2024-11-16T14:32:07,032 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplayValueCompression timeout: 13 mins 2024-11-16T14:32:07,053 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-16T14:32:07,093 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-16T14:32:07,093 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-16T14:32:07,104 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=3, rsPorts=, rsClass=null, numDataNodes=3, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-16T14:32:07,118 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/650842a5-0ab8-4683-35e8-38af23b31d67/cluster_174e5b33-598d-20b3-a770-4fb31341c285, deleteOnExit=true 2024-11-16T14:32:07,119 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-16T14:32:07,119 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/650842a5-0ab8-4683-35e8-38af23b31d67/test.cache.data in system properties and HBase conf 2024-11-16T14:32:07,120 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/650842a5-0ab8-4683-35e8-38af23b31d67/hadoop.tmp.dir in system properties and HBase conf 2024-11-16T14:32:07,121 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/650842a5-0ab8-4683-35e8-38af23b31d67/hadoop.log.dir in system properties and HBase conf 2024-11-16T14:32:07,121 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/650842a5-0ab8-4683-35e8-38af23b31d67/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-16T14:32:07,122 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/650842a5-0ab8-4683-35e8-38af23b31d67/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-16T14:32:07,122 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-16T14:32:07,191 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-16T14:32:07,276 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-16T14:32:07,280 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/650842a5-0ab8-4683-35e8-38af23b31d67/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-16T14:32:07,281 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/650842a5-0ab8-4683-35e8-38af23b31d67/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-16T14:32:07,281 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/650842a5-0ab8-4683-35e8-38af23b31d67/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-16T14:32:07,282 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/650842a5-0ab8-4683-35e8-38af23b31d67/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-16T14:32:07,282 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/650842a5-0ab8-4683-35e8-38af23b31d67/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-16T14:32:07,282 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/650842a5-0ab8-4683-35e8-38af23b31d67/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-16T14:32:07,283 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/650842a5-0ab8-4683-35e8-38af23b31d67/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-16T14:32:07,283 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/650842a5-0ab8-4683-35e8-38af23b31d67/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-16T14:32:07,284 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/650842a5-0ab8-4683-35e8-38af23b31d67/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-16T14:32:07,284 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/650842a5-0ab8-4683-35e8-38af23b31d67/nfs.dump.dir in system properties and HBase conf 2024-11-16T14:32:07,284 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/650842a5-0ab8-4683-35e8-38af23b31d67/java.io.tmpdir in system properties and HBase conf 2024-11-16T14:32:07,285 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/650842a5-0ab8-4683-35e8-38af23b31d67/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-16T14:32:07,285 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/650842a5-0ab8-4683-35e8-38af23b31d67/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-16T14:32:07,286 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/650842a5-0ab8-4683-35e8-38af23b31d67/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-16T14:32:08,366 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-16T14:32:08,434 INFO [Time-limited test {}] log.Log(170): Logging initialized @2474ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-16T14:32:08,501 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-16T14:32:08,561 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-16T14:32:08,585 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-16T14:32:08,585 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-16T14:32:08,586 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-16T14:32:08,601 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-16T14:32:08,605 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4f37ffca{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/650842a5-0ab8-4683-35e8-38af23b31d67/hadoop.log.dir/,AVAILABLE} 2024-11-16T14:32:08,606 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6dc9d5c1{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-16T14:32:08,779 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@3717288f{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/650842a5-0ab8-4683-35e8-38af23b31d67/java.io.tmpdir/jetty-localhost-40917-hadoop-hdfs-3_4_1-tests_jar-_-any-14567026366947695259/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-16T14:32:08,784 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@4bd70930{HTTP/1.1, (http/1.1)}{localhost:40917} 2024-11-16T14:32:08,784 INFO [Time-limited test {}] server.Server(415): Started @2825ms 2024-11-16T14:32:09,333 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-16T14:32:09,340 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-16T14:32:09,341 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-16T14:32:09,342 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-16T14:32:09,342 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-16T14:32:09,342 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@cf5a85e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/650842a5-0ab8-4683-35e8-38af23b31d67/hadoop.log.dir/,AVAILABLE} 2024-11-16T14:32:09,343 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4a359997{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-16T14:32:09,439 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@330740de{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/650842a5-0ab8-4683-35e8-38af23b31d67/java.io.tmpdir/jetty-localhost-41413-hadoop-hdfs-3_4_1-tests_jar-_-any-1611226531334857052/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-16T14:32:09,440 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@7b24cab9{HTTP/1.1, (http/1.1)}{localhost:41413} 2024-11-16T14:32:09,441 INFO [Time-limited test {}] server.Server(415): Started @3481ms 2024-11-16T14:32:09,484 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-16T14:32:09,594 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-16T14:32:09,599 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-16T14:32:09,603 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-16T14:32:09,603 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-16T14:32:09,603 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-16T14:32:09,605 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@46b092e1{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/650842a5-0ab8-4683-35e8-38af23b31d67/hadoop.log.dir/,AVAILABLE} 2024-11-16T14:32:09,606 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@5cc2d6b{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-16T14:32:09,713 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@7bd427b8{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/650842a5-0ab8-4683-35e8-38af23b31d67/java.io.tmpdir/jetty-localhost-40289-hadoop-hdfs-3_4_1-tests_jar-_-any-17997043025570137713/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-16T14:32:09,714 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@6915083f{HTTP/1.1, (http/1.1)}{localhost:40289} 2024-11-16T14:32:09,714 INFO [Time-limited test {}] server.Server(415): Started @3755ms 2024-11-16T14:32:09,716 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-16T14:32:09,749 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-16T14:32:09,754 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-16T14:32:09,755 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-16T14:32:09,755 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-16T14:32:09,756 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-16T14:32:09,756 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@2a6d5e13{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/650842a5-0ab8-4683-35e8-38af23b31d67/hadoop.log.dir/,AVAILABLE} 2024-11-16T14:32:09,757 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@f9972d0{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-16T14:32:09,853 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@35f1150e{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/650842a5-0ab8-4683-35e8-38af23b31d67/java.io.tmpdir/jetty-localhost-34059-hadoop-hdfs-3_4_1-tests_jar-_-any-14282502564959074146/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-16T14:32:09,854 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@13a77e13{HTTP/1.1, (http/1.1)}{localhost:34059} 2024-11-16T14:32:09,854 INFO [Time-limited test {}] server.Server(415): Started @3895ms 2024-11-16T14:32:09,856 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-16T14:32:11,025 WARN [Thread-122 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/650842a5-0ab8-4683-35e8-38af23b31d67/cluster_174e5b33-598d-20b3-a770-4fb31341c285/data/data3/current/BP-865919460-172.17.0.2-1731767527819/current, will proceed with Du for space computation calculation, 2024-11-16T14:32:11,025 WARN [Thread-125 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/650842a5-0ab8-4683-35e8-38af23b31d67/cluster_174e5b33-598d-20b3-a770-4fb31341c285/data/data2/current/BP-865919460-172.17.0.2-1731767527819/current, will proceed with Du for space computation calculation, 2024-11-16T14:32:11,025 WARN [Thread-124 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/650842a5-0ab8-4683-35e8-38af23b31d67/cluster_174e5b33-598d-20b3-a770-4fb31341c285/data/data1/current/BP-865919460-172.17.0.2-1731767527819/current, will proceed with Du for space computation calculation, 2024-11-16T14:32:11,025 WARN [Thread-123 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/650842a5-0ab8-4683-35e8-38af23b31d67/cluster_174e5b33-598d-20b3-a770-4fb31341c285/data/data4/current/BP-865919460-172.17.0.2-1731767527819/current, will proceed with Du for space computation calculation, 2024-11-16T14:32:11,053 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-16T14:32:11,053 WARN [Thread-81 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-16T14:32:11,101 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x91a5390187a92497 with lease ID 0x4c47d504f2c2f8d6: Processing first storage report for DS-17ab9554-4509-49b3-918b-178692b9a473 from datanode DatanodeRegistration(127.0.0.1:43279, datanodeUuid=870c4796-08a6-4764-94fb-ce871302a15a, infoPort=36015, infoSecurePort=0, ipcPort=39925, storageInfo=lv=-57;cid=testClusterID;nsid=1747753455;c=1731767527819) 2024-11-16T14:32:11,102 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x91a5390187a92497 with lease ID 0x4c47d504f2c2f8d6: from storage DS-17ab9554-4509-49b3-918b-178692b9a473 node DatanodeRegistration(127.0.0.1:43279, datanodeUuid=870c4796-08a6-4764-94fb-ce871302a15a, infoPort=36015, infoSecurePort=0, ipcPort=39925, storageInfo=lv=-57;cid=testClusterID;nsid=1747753455;c=1731767527819), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-16T14:32:11,102 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x96ca05a37c143b42 with lease ID 0x4c47d504f2c2f8d5: Processing first storage report for DS-6a1dd528-c0be-4580-a53e-7113879f7f97 from datanode DatanodeRegistration(127.0.0.1:33005, datanodeUuid=da003da4-b43a-4f8f-a16e-388e885fb77e, infoPort=41179, infoSecurePort=0, ipcPort=41505, storageInfo=lv=-57;cid=testClusterID;nsid=1747753455;c=1731767527819) 2024-11-16T14:32:11,102 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x96ca05a37c143b42 with lease ID 0x4c47d504f2c2f8d5: from storage DS-6a1dd528-c0be-4580-a53e-7113879f7f97 node DatanodeRegistration(127.0.0.1:33005, datanodeUuid=da003da4-b43a-4f8f-a16e-388e885fb77e, infoPort=41179, infoSecurePort=0, ipcPort=41505, storageInfo=lv=-57;cid=testClusterID;nsid=1747753455;c=1731767527819), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-16T14:32:11,102 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x91a5390187a92497 with lease ID 0x4c47d504f2c2f8d6: Processing first storage report for DS-2938cd27-0254-4dc2-b25c-f1ef1f0c21a3 from datanode DatanodeRegistration(127.0.0.1:43279, datanodeUuid=870c4796-08a6-4764-94fb-ce871302a15a, infoPort=36015, infoSecurePort=0, ipcPort=39925, storageInfo=lv=-57;cid=testClusterID;nsid=1747753455;c=1731767527819) 2024-11-16T14:32:11,103 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x91a5390187a92497 with lease ID 0x4c47d504f2c2f8d6: from storage DS-2938cd27-0254-4dc2-b25c-f1ef1f0c21a3 node DatanodeRegistration(127.0.0.1:43279, datanodeUuid=870c4796-08a6-4764-94fb-ce871302a15a, infoPort=36015, infoSecurePort=0, ipcPort=39925, storageInfo=lv=-57;cid=testClusterID;nsid=1747753455;c=1731767527819), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-16T14:32:11,103 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x96ca05a37c143b42 with lease ID 0x4c47d504f2c2f8d5: Processing first storage report for DS-302e899a-1cfc-4db0-8604-690d8cda3c78 from datanode DatanodeRegistration(127.0.0.1:33005, datanodeUuid=da003da4-b43a-4f8f-a16e-388e885fb77e, infoPort=41179, infoSecurePort=0, ipcPort=41505, storageInfo=lv=-57;cid=testClusterID;nsid=1747753455;c=1731767527819) 2024-11-16T14:32:11,103 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x96ca05a37c143b42 with lease ID 0x4c47d504f2c2f8d5: from storage DS-302e899a-1cfc-4db0-8604-690d8cda3c78 node DatanodeRegistration(127.0.0.1:33005, datanodeUuid=da003da4-b43a-4f8f-a16e-388e885fb77e, infoPort=41179, infoSecurePort=0, ipcPort=41505, storageInfo=lv=-57;cid=testClusterID;nsid=1747753455;c=1731767527819), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-16T14:32:11,230 WARN [Thread-143 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/650842a5-0ab8-4683-35e8-38af23b31d67/cluster_174e5b33-598d-20b3-a770-4fb31341c285/data/data6/current/BP-865919460-172.17.0.2-1731767527819/current, will proceed with Du for space computation calculation, 2024-11-16T14:32:11,230 WARN [Thread-142 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/650842a5-0ab8-4683-35e8-38af23b31d67/cluster_174e5b33-598d-20b3-a770-4fb31341c285/data/data5/current/BP-865919460-172.17.0.2-1731767527819/current, will proceed with Du for space computation calculation, 2024-11-16T14:32:11,254 WARN [Thread-103 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-16T14:32:11,258 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xbb762eecfc22adda with lease ID 0x4c47d504f2c2f8d7: Processing first storage report for DS-a30e140c-8532-4096-8cae-c8d713a3d4e3 from datanode DatanodeRegistration(127.0.0.1:36193, datanodeUuid=a4e7ac56-98d1-4613-9843-f3379d05420b, infoPort=35677, infoSecurePort=0, ipcPort=43019, storageInfo=lv=-57;cid=testClusterID;nsid=1747753455;c=1731767527819) 2024-11-16T14:32:11,259 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xbb762eecfc22adda with lease ID 0x4c47d504f2c2f8d7: from storage DS-a30e140c-8532-4096-8cae-c8d713a3d4e3 node DatanodeRegistration(127.0.0.1:36193, datanodeUuid=a4e7ac56-98d1-4613-9843-f3379d05420b, infoPort=35677, infoSecurePort=0, ipcPort=43019, storageInfo=lv=-57;cid=testClusterID;nsid=1747753455;c=1731767527819), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-16T14:32:11,259 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xbb762eecfc22adda with lease ID 0x4c47d504f2c2f8d7: Processing first storage report for DS-57178ba6-d3be-4682-ba49-cdb263f06fcd from datanode DatanodeRegistration(127.0.0.1:36193, datanodeUuid=a4e7ac56-98d1-4613-9843-f3379d05420b, infoPort=35677, infoSecurePort=0, ipcPort=43019, storageInfo=lv=-57;cid=testClusterID;nsid=1747753455;c=1731767527819) 2024-11-16T14:32:11,259 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xbb762eecfc22adda with lease ID 0x4c47d504f2c2f8d7: from storage DS-57178ba6-d3be-4682-ba49-cdb263f06fcd node DatanodeRegistration(127.0.0.1:36193, datanodeUuid=a4e7ac56-98d1-4613-9843-f3379d05420b, infoPort=35677, infoSecurePort=0, ipcPort=43019, storageInfo=lv=-57;cid=testClusterID;nsid=1747753455;c=1731767527819), blocks: 0, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-16T14:32:11,265 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/650842a5-0ab8-4683-35e8-38af23b31d67 2024-11-16T14:32:11,325 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/650842a5-0ab8-4683-35e8-38af23b31d67/cluster_174e5b33-598d-20b3-a770-4fb31341c285/zookeeper_0, clientPort=57212, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/650842a5-0ab8-4683-35e8-38af23b31d67/cluster_174e5b33-598d-20b3-a770-4fb31341c285/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/650842a5-0ab8-4683-35e8-38af23b31d67/cluster_174e5b33-598d-20b3-a770-4fb31341c285/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-16T14:32:11,344 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=57212 2024-11-16T14:32:11,356 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-16T14:32:11,358 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-16T14:32:11,551 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36193 is added to blk_1073741825_1001 (size=7) 2024-11-16T14:32:11,552 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33005 is added to blk_1073741825_1001 (size=7) 2024-11-16T14:32:11,552 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43279 is added to blk_1073741825_1001 (size=7) 2024-11-16T14:32:11,968 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:43457/user/jenkins/test-data/0d13f5ec-97ce-91d4-7824-c5174956db56 with version=8 2024-11-16T14:32:11,969 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:43457/user/jenkins/test-data/0d13f5ec-97ce-91d4-7824-c5174956db56/hbase-staging 2024-11-16T14:32:12,241 INFO [Time-limited test {}] client.ConnectionUtils(128): master/04e93f2b5fad:0 server-side Connection retries=45 2024-11-16T14:32:12,249 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-16T14:32:12,250 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-16T14:32:12,254 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-16T14:32:12,255 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-16T14:32:12,255 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-16T14:32:12,381 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-16T14:32:12,434 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-16T14:32:12,443 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-16T14:32:12,446 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T14:32:12,470 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 98010 (auto-detected) 2024-11-16T14:32:12,471 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-16T14:32:12,489 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:34365 2024-11-16T14:32:12,507 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:34365 connecting to ZooKeeper ensemble=127.0.0.1:57212 2024-11-16T14:32:12,631 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:343650x0, quorum=127.0.0.1:57212, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-16T14:32:12,633 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:34365-0x101440f72da0000 connected 2024-11-16T14:32:12,760 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-16T14:32:12,766 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-16T14:32:12,779 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:34365-0x101440f72da0000, quorum=127.0.0.1:57212, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-16T14:32:12,784 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:43457/user/jenkins/test-data/0d13f5ec-97ce-91d4-7824-c5174956db56, hbase.cluster.distributed=false 2024-11-16T14:32:12,807 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:34365-0x101440f72da0000, quorum=127.0.0.1:57212, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-16T14:32:12,812 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=34365 2024-11-16T14:32:12,813 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=34365 2024-11-16T14:32:12,814 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=34365 2024-11-16T14:32:12,814 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=34365 2024-11-16T14:32:12,815 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=34365 2024-11-16T14:32:12,903 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/04e93f2b5fad:0 server-side Connection retries=45 2024-11-16T14:32:12,905 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-16T14:32:12,905 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-16T14:32:12,905 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-16T14:32:12,905 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-16T14:32:12,905 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-16T14:32:12,908 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-16T14:32:12,910 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T14:32:12,912 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:35241 2024-11-16T14:32:12,913 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:35241 connecting to ZooKeeper ensemble=127.0.0.1:57212 2024-11-16T14:32:12,914 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-16T14:32:12,919 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-16T14:32:12,934 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:352410x0, quorum=127.0.0.1:57212, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-16T14:32:12,935 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:35241-0x101440f72da0001 connected 2024-11-16T14:32:12,935 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:35241-0x101440f72da0001, quorum=127.0.0.1:57212, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-16T14:32:12,938 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-16T14:32:12,946 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-16T14:32:12,948 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:35241-0x101440f72da0001, quorum=127.0.0.1:57212, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-16T14:32:12,953 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:35241-0x101440f72da0001, quorum=127.0.0.1:57212, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-16T14:32:12,954 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=35241 2024-11-16T14:32:12,954 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=35241 2024-11-16T14:32:12,954 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=35241 2024-11-16T14:32:12,955 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=35241 2024-11-16T14:32:12,955 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=35241 2024-11-16T14:32:12,971 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/04e93f2b5fad:0 server-side Connection retries=45 2024-11-16T14:32:12,971 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-16T14:32:12,971 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-16T14:32:12,971 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-16T14:32:12,972 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-16T14:32:12,972 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-16T14:32:12,972 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-16T14:32:12,972 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T14:32:12,973 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:39921 2024-11-16T14:32:12,975 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:39921 connecting to ZooKeeper ensemble=127.0.0.1:57212 2024-11-16T14:32:12,975 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-16T14:32:12,979 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-16T14:32:12,992 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:399210x0, quorum=127.0.0.1:57212, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-16T14:32:12,993 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:399210x0, quorum=127.0.0.1:57212, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-16T14:32:12,993 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:39921-0x101440f72da0002 connected 2024-11-16T14:32:12,993 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-16T14:32:12,994 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-16T14:32:12,995 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:39921-0x101440f72da0002, quorum=127.0.0.1:57212, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-16T14:32:12,997 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:39921-0x101440f72da0002, quorum=127.0.0.1:57212, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-16T14:32:13,000 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=39921 2024-11-16T14:32:13,001 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=39921 2024-11-16T14:32:13,001 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=39921 2024-11-16T14:32:13,002 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=39921 2024-11-16T14:32:13,002 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=39921 2024-11-16T14:32:13,018 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/04e93f2b5fad:0 server-side Connection retries=45 2024-11-16T14:32:13,018 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-16T14:32:13,018 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-16T14:32:13,018 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-16T14:32:13,019 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-16T14:32:13,019 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-16T14:32:13,019 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-16T14:32:13,019 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T14:32:13,020 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:45119 2024-11-16T14:32:13,022 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:45119 connecting to ZooKeeper ensemble=127.0.0.1:57212 2024-11-16T14:32:13,023 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-16T14:32:13,025 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-16T14:32:13,039 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:451190x0, quorum=127.0.0.1:57212, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-16T14:32:13,040 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:45119-0x101440f72da0003 connected 2024-11-16T14:32:13,040 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:45119-0x101440f72da0003, quorum=127.0.0.1:57212, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-16T14:32:13,041 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-16T14:32:13,042 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-16T14:32:13,043 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:45119-0x101440f72da0003, quorum=127.0.0.1:57212, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-16T14:32:13,045 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:45119-0x101440f72da0003, quorum=127.0.0.1:57212, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-16T14:32:13,045 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=45119 2024-11-16T14:32:13,046 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=45119 2024-11-16T14:32:13,047 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=45119 2024-11-16T14:32:13,048 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=45119 2024-11-16T14:32:13,052 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=45119 2024-11-16T14:32:13,065 DEBUG [M:0;04e93f2b5fad:34365 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;04e93f2b5fad:34365 2024-11-16T14:32:13,066 INFO [master/04e93f2b5fad:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/04e93f2b5fad,34365,1731767532092 2024-11-16T14:32:13,082 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45119-0x101440f72da0003, quorum=127.0.0.1:57212, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-16T14:32:13,082 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39921-0x101440f72da0002, quorum=127.0.0.1:57212, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-16T14:32:13,082 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35241-0x101440f72da0001, quorum=127.0.0.1:57212, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-16T14:32:13,082 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34365-0x101440f72da0000, quorum=127.0.0.1:57212, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-16T14:32:13,084 DEBUG [master/04e93f2b5fad:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:34365-0x101440f72da0000, quorum=127.0.0.1:57212, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/04e93f2b5fad,34365,1731767532092 2024-11-16T14:32:13,113 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35241-0x101440f72da0001, quorum=127.0.0.1:57212, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-16T14:32:13,113 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45119-0x101440f72da0003, quorum=127.0.0.1:57212, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-16T14:32:13,113 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39921-0x101440f72da0002, quorum=127.0.0.1:57212, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-16T14:32:13,113 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35241-0x101440f72da0001, quorum=127.0.0.1:57212, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T14:32:13,113 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34365-0x101440f72da0000, quorum=127.0.0.1:57212, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T14:32:13,113 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39921-0x101440f72da0002, quorum=127.0.0.1:57212, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T14:32:13,114 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45119-0x101440f72da0003, quorum=127.0.0.1:57212, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T14:32:13,114 DEBUG [master/04e93f2b5fad:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:34365-0x101440f72da0000, quorum=127.0.0.1:57212, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-16T14:32:13,116 INFO [master/04e93f2b5fad:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/04e93f2b5fad,34365,1731767532092 from backup master directory 2024-11-16T14:32:13,129 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45119-0x101440f72da0003, quorum=127.0.0.1:57212, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-16T14:32:13,129 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34365-0x101440f72da0000, quorum=127.0.0.1:57212, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/04e93f2b5fad,34365,1731767532092 2024-11-16T14:32:13,129 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39921-0x101440f72da0002, quorum=127.0.0.1:57212, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-16T14:32:13,129 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35241-0x101440f72da0001, quorum=127.0.0.1:57212, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-16T14:32:13,129 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34365-0x101440f72da0000, quorum=127.0.0.1:57212, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-16T14:32:13,130 WARN [master/04e93f2b5fad:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-16T14:32:13,131 INFO [master/04e93f2b5fad:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=04e93f2b5fad,34365,1731767532092 2024-11-16T14:32:13,133 INFO [master/04e93f2b5fad:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-16T14:32:13,135 INFO [master/04e93f2b5fad:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-16T14:32:13,188 DEBUG [master/04e93f2b5fad:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:43457/user/jenkins/test-data/0d13f5ec-97ce-91d4-7824-c5174956db56/hbase.id] with ID: 6ead4d3f-84d5-440b-9e8d-d5bc0db5a8c1 2024-11-16T14:32:13,188 DEBUG [master/04e93f2b5fad:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:43457/user/jenkins/test-data/0d13f5ec-97ce-91d4-7824-c5174956db56/.tmp/hbase.id 2024-11-16T14:32:13,201 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36193 is added to blk_1073741826_1002 (size=42) 2024-11-16T14:32:13,201 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33005 is added to blk_1073741826_1002 (size=42) 2024-11-16T14:32:13,202 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43279 is added to blk_1073741826_1002 (size=42) 2024-11-16T14:32:13,203 DEBUG [master/04e93f2b5fad:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:43457/user/jenkins/test-data/0d13f5ec-97ce-91d4-7824-c5174956db56/.tmp/hbase.id]:[hdfs://localhost:43457/user/jenkins/test-data/0d13f5ec-97ce-91d4-7824-c5174956db56/hbase.id] 2024-11-16T14:32:13,247 INFO [master/04e93f2b5fad:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-16T14:32:13,251 INFO [master/04e93f2b5fad:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-16T14:32:13,269 INFO [master/04e93f2b5fad:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 16ms. 2024-11-16T14:32:13,281 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34365-0x101440f72da0000, quorum=127.0.0.1:57212, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T14:32:13,281 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35241-0x101440f72da0001, quorum=127.0.0.1:57212, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T14:32:13,281 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45119-0x101440f72da0003, quorum=127.0.0.1:57212, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T14:32:13,281 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39921-0x101440f72da0002, quorum=127.0.0.1:57212, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T14:32:13,302 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33005 is added to blk_1073741827_1003 (size=196) 2024-11-16T14:32:13,302 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36193 is added to blk_1073741827_1003 (size=196) 2024-11-16T14:32:13,302 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43279 is added to blk_1073741827_1003 (size=196) 2024-11-16T14:32:13,317 INFO [master/04e93f2b5fad:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-16T14:32:13,318 INFO [master/04e93f2b5fad:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-16T14:32:13,323 INFO [master/04e93f2b5fad:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-16T14:32:13,354 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33005 is added to blk_1073741828_1004 (size=1189) 2024-11-16T14:32:13,355 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43279 is added to blk_1073741828_1004 (size=1189) 2024-11-16T14:32:13,355 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36193 is added to blk_1073741828_1004 (size=1189) 2024-11-16T14:32:13,370 INFO [master/04e93f2b5fad:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:43457/user/jenkins/test-data/0d13f5ec-97ce-91d4-7824-c5174956db56/MasterData/data/master/store 2024-11-16T14:32:13,392 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33005 is added to blk_1073741829_1005 (size=34) 2024-11-16T14:32:13,392 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36193 is added to blk_1073741829_1005 (size=34) 2024-11-16T14:32:13,392 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43279 is added to blk_1073741829_1005 (size=34) 2024-11-16T14:32:13,397 INFO [master/04e93f2b5fad:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-16T14:32:13,400 DEBUG [master/04e93f2b5fad:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T14:32:13,401 DEBUG [master/04e93f2b5fad:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-16T14:32:13,401 INFO [master/04e93f2b5fad:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-16T14:32:13,401 DEBUG [master/04e93f2b5fad:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-16T14:32:13,403 DEBUG [master/04e93f2b5fad:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-16T14:32:13,403 DEBUG [master/04e93f2b5fad:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-16T14:32:13,403 INFO [master/04e93f2b5fad:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-16T14:32:13,404 DEBUG [master/04e93f2b5fad:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731767533401Disabling compacts and flushes for region at 1731767533401Disabling writes for close at 1731767533403 (+2 ms)Writing region close event to WAL at 1731767533403Closed at 1731767533403 2024-11-16T14:32:13,406 WARN [master/04e93f2b5fad:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:43457/user/jenkins/test-data/0d13f5ec-97ce-91d4-7824-c5174956db56/MasterData/data/master/store/.initializing 2024-11-16T14:32:13,406 DEBUG [master/04e93f2b5fad:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:43457/user/jenkins/test-data/0d13f5ec-97ce-91d4-7824-c5174956db56/MasterData/WALs/04e93f2b5fad,34365,1731767532092 2024-11-16T14:32:13,413 INFO [master/04e93f2b5fad:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-16T14:32:13,427 INFO [master/04e93f2b5fad:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=04e93f2b5fad%2C34365%2C1731767532092, suffix=, logDir=hdfs://localhost:43457/user/jenkins/test-data/0d13f5ec-97ce-91d4-7824-c5174956db56/MasterData/WALs/04e93f2b5fad,34365,1731767532092, archiveDir=hdfs://localhost:43457/user/jenkins/test-data/0d13f5ec-97ce-91d4-7824-c5174956db56/MasterData/oldWALs, maxLogs=10 2024-11-16T14:32:13,454 DEBUG [master/04e93f2b5fad:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/0d13f5ec-97ce-91d4-7824-c5174956db56/MasterData/WALs/04e93f2b5fad,34365,1731767532092/04e93f2b5fad%2C34365%2C1731767532092.1731767533431, exclude list is [], retry=0 2024-11-16T14:32:13,472 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.initialize(FanOutOneBlockAsyncDFSOutputHelper.java:413) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$5.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:472) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$5.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:467) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.NettyFutureUtils.lambda$addListener$0(NettyFutureUtils.java:56) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setSuccess0(DefaultPromise.java:625) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.trySuccess(DefaultPromise.java:105) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPromise.trySuccess(DefaultChannelPromise.java:84) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:658) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:696) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:32:13,476 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33005,DS-6a1dd528-c0be-4580-a53e-7113879f7f97,DISK] 2024-11-16T14:32:13,476 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36193,DS-a30e140c-8532-4096-8cae-c8d713a3d4e3,DISK] 2024-11-16T14:32:13,476 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43279,DS-17ab9554-4509-49b3-918b-178692b9a473,DISK] 2024-11-16T14:32:13,479 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-16T14:32:13,515 INFO [master/04e93f2b5fad:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/0d13f5ec-97ce-91d4-7824-c5174956db56/MasterData/WALs/04e93f2b5fad,34365,1731767532092/04e93f2b5fad%2C34365%2C1731767532092.1731767533431 2024-11-16T14:32:13,516 DEBUG [master/04e93f2b5fad:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35677:35677),(127.0.0.1/127.0.0.1:36015:36015),(127.0.0.1/127.0.0.1:41179:41179)] 2024-11-16T14:32:13,516 DEBUG [master/04e93f2b5fad:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-16T14:32:13,517 DEBUG [master/04e93f2b5fad:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T14:32:13,520 DEBUG [master/04e93f2b5fad:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-16T14:32:13,521 DEBUG [master/04e93f2b5fad:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-16T14:32:13,553 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-16T14:32:13,576 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-16T14:32:13,579 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:32:13,582 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-16T14:32:13,582 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-16T14:32:13,586 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-16T14:32:13,586 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:32:13,587 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T14:32:13,588 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-16T14:32:13,591 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-16T14:32:13,592 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:32:13,593 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T14:32:13,593 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-16T14:32:13,597 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-16T14:32:13,597 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:32:13,598 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T14:32:13,599 DEBUG [master/04e93f2b5fad:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-16T14:32:13,602 DEBUG [master/04e93f2b5fad:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43457/user/jenkins/test-data/0d13f5ec-97ce-91d4-7824-c5174956db56/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-16T14:32:13,604 DEBUG [master/04e93f2b5fad:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43457/user/jenkins/test-data/0d13f5ec-97ce-91d4-7824-c5174956db56/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-16T14:32:13,609 DEBUG [master/04e93f2b5fad:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-16T14:32:13,609 DEBUG [master/04e93f2b5fad:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-16T14:32:13,613 DEBUG [master/04e93f2b5fad:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-16T14:32:13,616 DEBUG [master/04e93f2b5fad:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-16T14:32:13,620 DEBUG [master/04e93f2b5fad:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43457/user/jenkins/test-data/0d13f5ec-97ce-91d4-7824-c5174956db56/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-16T14:32:13,622 INFO [master/04e93f2b5fad:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68901471, jitterRate=0.026711925864219666}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-16T14:32:13,628 DEBUG [master/04e93f2b5fad:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731767533532Initializing all the Stores at 1731767533534 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731767533534Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731767533535 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731767533535Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731767533535Cleaning up temporary data from old regions at 1731767533609 (+74 ms)Region opened successfully at 1731767533628 (+19 ms) 2024-11-16T14:32:13,629 INFO [master/04e93f2b5fad:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-16T14:32:13,658 DEBUG [master/04e93f2b5fad:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b472f40, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=04e93f2b5fad/172.17.0.2:0 2024-11-16T14:32:13,684 INFO [master/04e93f2b5fad:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-16T14:32:13,693 INFO [master/04e93f2b5fad:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-16T14:32:13,693 INFO [master/04e93f2b5fad:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-16T14:32:13,695 INFO [master/04e93f2b5fad:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-16T14:32:13,696 INFO [master/04e93f2b5fad:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-16T14:32:13,700 INFO [master/04e93f2b5fad:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 3 msec 2024-11-16T14:32:13,700 INFO [master/04e93f2b5fad:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-16T14:32:13,723 INFO [master/04e93f2b5fad:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-16T14:32:13,730 DEBUG [master/04e93f2b5fad:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34365-0x101440f72da0000, quorum=127.0.0.1:57212, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-16T14:32:13,781 DEBUG [master/04e93f2b5fad:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-16T14:32:13,783 INFO [master/04e93f2b5fad:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-16T14:32:13,785 DEBUG [master/04e93f2b5fad:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34365-0x101440f72da0000, quorum=127.0.0.1:57212, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-16T14:32:13,797 DEBUG [master/04e93f2b5fad:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-16T14:32:13,799 INFO [master/04e93f2b5fad:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-16T14:32:13,804 DEBUG [master/04e93f2b5fad:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34365-0x101440f72da0000, quorum=127.0.0.1:57212, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-16T14:32:13,812 DEBUG [master/04e93f2b5fad:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-16T14:32:13,814 DEBUG [master/04e93f2b5fad:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34365-0x101440f72da0000, quorum=127.0.0.1:57212, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-16T14:32:13,823 DEBUG [master/04e93f2b5fad:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-16T14:32:13,845 DEBUG [master/04e93f2b5fad:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34365-0x101440f72da0000, quorum=127.0.0.1:57212, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-16T14:32:13,854 DEBUG [master/04e93f2b5fad:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-16T14:32:13,865 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35241-0x101440f72da0001, quorum=127.0.0.1:57212, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-16T14:32:13,865 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39921-0x101440f72da0002, quorum=127.0.0.1:57212, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-16T14:32:13,865 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45119-0x101440f72da0003, quorum=127.0.0.1:57212, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-16T14:32:13,865 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34365-0x101440f72da0000, quorum=127.0.0.1:57212, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-16T14:32:13,866 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35241-0x101440f72da0001, quorum=127.0.0.1:57212, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T14:32:13,866 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45119-0x101440f72da0003, quorum=127.0.0.1:57212, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T14:32:13,866 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39921-0x101440f72da0002, quorum=127.0.0.1:57212, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T14:32:13,866 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34365-0x101440f72da0000, quorum=127.0.0.1:57212, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T14:32:13,870 INFO [master/04e93f2b5fad:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=04e93f2b5fad,34365,1731767532092, sessionid=0x101440f72da0000, setting cluster-up flag (Was=false) 2024-11-16T14:32:13,903 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45119-0x101440f72da0003, quorum=127.0.0.1:57212, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T14:32:13,903 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39921-0x101440f72da0002, quorum=127.0.0.1:57212, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T14:32:13,903 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34365-0x101440f72da0000, quorum=127.0.0.1:57212, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T14:32:13,903 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35241-0x101440f72da0001, quorum=127.0.0.1:57212, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T14:32:13,940 DEBUG [master/04e93f2b5fad:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-16T14:32:13,947 DEBUG [master/04e93f2b5fad:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=04e93f2b5fad,34365,1731767532092 2024-11-16T14:32:13,971 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35241-0x101440f72da0001, quorum=127.0.0.1:57212, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T14:32:13,971 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34365-0x101440f72da0000, quorum=127.0.0.1:57212, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T14:32:13,971 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39921-0x101440f72da0002, quorum=127.0.0.1:57212, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T14:32:13,971 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45119-0x101440f72da0003, quorum=127.0.0.1:57212, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T14:32:14,002 DEBUG [master/04e93f2b5fad:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-16T14:32:14,004 DEBUG [master/04e93f2b5fad:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=04e93f2b5fad,34365,1731767532092 2024-11-16T14:32:14,010 INFO [master/04e93f2b5fad:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:43457/user/jenkins/test-data/0d13f5ec-97ce-91d4-7824-c5174956db56/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-16T14:32:14,056 INFO [RS:0;04e93f2b5fad:35241 {}] regionserver.HRegionServer(746): ClusterId : 6ead4d3f-84d5-440b-9e8d-d5bc0db5a8c1 2024-11-16T14:32:14,056 INFO [RS:1;04e93f2b5fad:39921 {}] regionserver.HRegionServer(746): ClusterId : 6ead4d3f-84d5-440b-9e8d-d5bc0db5a8c1 2024-11-16T14:32:14,056 INFO [RS:2;04e93f2b5fad:45119 {}] regionserver.HRegionServer(746): ClusterId : 6ead4d3f-84d5-440b-9e8d-d5bc0db5a8c1 2024-11-16T14:32:14,058 DEBUG [RS:0;04e93f2b5fad:35241 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-16T14:32:14,058 DEBUG [RS:1;04e93f2b5fad:39921 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-16T14:32:14,058 DEBUG [RS:2;04e93f2b5fad:45119 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-16T14:32:14,083 DEBUG [RS:2;04e93f2b5fad:45119 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-16T14:32:14,083 DEBUG [RS:1;04e93f2b5fad:39921 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-16T14:32:14,083 DEBUG [RS:0;04e93f2b5fad:35241 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-16T14:32:14,083 DEBUG [RS:2;04e93f2b5fad:45119 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-16T14:32:14,083 DEBUG [RS:1;04e93f2b5fad:39921 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-16T14:32:14,083 DEBUG [RS:0;04e93f2b5fad:35241 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-16T14:32:14,085 INFO [AsyncFSWAL-0-hdfs://localhost:43457/user/jenkins/test-data/0d13f5ec-97ce-91d4-7824-c5174956db56/MasterData-prefix:04e93f2b5fad,34365,1731767532092 {}] compress.Compression(560): Loaded codec org.apache.hadoop.hbase.io.compress.ReusableStreamGzipCodec for compression algorithm GZ 2024-11-16T14:32:14,100 DEBUG [master/04e93f2b5fad:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-16T14:32:14,103 DEBUG [RS:0;04e93f2b5fad:35241 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-16T14:32:14,103 DEBUG [RS:2;04e93f2b5fad:45119 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-16T14:32:14,104 DEBUG [RS:1;04e93f2b5fad:39921 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-16T14:32:14,104 DEBUG [RS:0;04e93f2b5fad:35241 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ead01eb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=04e93f2b5fad/172.17.0.2:0 2024-11-16T14:32:14,104 DEBUG [RS:2;04e93f2b5fad:45119 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@395da7ac, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=04e93f2b5fad/172.17.0.2:0 2024-11-16T14:32:14,104 DEBUG [RS:1;04e93f2b5fad:39921 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59af3856, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=04e93f2b5fad/172.17.0.2:0 2024-11-16T14:32:14,110 INFO [master/04e93f2b5fad:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-16T14:32:14,118 INFO [master/04e93f2b5fad:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-16T14:32:14,123 DEBUG [RS:2;04e93f2b5fad:45119 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:2;04e93f2b5fad:45119 2024-11-16T14:32:14,123 DEBUG [RS:0;04e93f2b5fad:35241 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;04e93f2b5fad:35241 2024-11-16T14:32:14,123 DEBUG [RS:1;04e93f2b5fad:39921 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;04e93f2b5fad:39921 2024-11-16T14:32:14,126 INFO [RS:0;04e93f2b5fad:35241 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-16T14:32:14,126 INFO [RS:2;04e93f2b5fad:45119 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-16T14:32:14,126 INFO [RS:1;04e93f2b5fad:39921 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-16T14:32:14,126 INFO [RS:0;04e93f2b5fad:35241 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-16T14:32:14,126 INFO [RS:2;04e93f2b5fad:45119 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-16T14:32:14,126 INFO [RS:1;04e93f2b5fad:39921 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-16T14:32:14,126 DEBUG [RS:2;04e93f2b5fad:45119 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-16T14:32:14,126 DEBUG [RS:0;04e93f2b5fad:35241 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-16T14:32:14,126 DEBUG [RS:1;04e93f2b5fad:39921 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-16T14:32:14,125 DEBUG [master/04e93f2b5fad:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 04e93f2b5fad,34365,1731767532092 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-16T14:32:14,129 INFO [RS:0;04e93f2b5fad:35241 {}] regionserver.HRegionServer(2659): reportForDuty to master=04e93f2b5fad,34365,1731767532092 with port=35241, startcode=1731767532873 2024-11-16T14:32:14,129 INFO [RS:2;04e93f2b5fad:45119 {}] regionserver.HRegionServer(2659): reportForDuty to master=04e93f2b5fad,34365,1731767532092 with port=45119, startcode=1731767533018 2024-11-16T14:32:14,129 INFO [RS:1;04e93f2b5fad:39921 {}] regionserver.HRegionServer(2659): reportForDuty to master=04e93f2b5fad,34365,1731767532092 with port=39921, startcode=1731767532970 2024-11-16T14:32:14,132 DEBUG [master/04e93f2b5fad:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/04e93f2b5fad:0, corePoolSize=5, maxPoolSize=5 2024-11-16T14:32:14,132 DEBUG [master/04e93f2b5fad:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/04e93f2b5fad:0, corePoolSize=5, maxPoolSize=5 2024-11-16T14:32:14,132 DEBUG [master/04e93f2b5fad:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/04e93f2b5fad:0, corePoolSize=5, maxPoolSize=5 2024-11-16T14:32:14,133 DEBUG [master/04e93f2b5fad:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/04e93f2b5fad:0, corePoolSize=5, maxPoolSize=5 2024-11-16T14:32:14,133 DEBUG [master/04e93f2b5fad:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/04e93f2b5fad:0, corePoolSize=10, maxPoolSize=10 2024-11-16T14:32:14,133 DEBUG [master/04e93f2b5fad:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/04e93f2b5fad:0, corePoolSize=1, maxPoolSize=1 2024-11-16T14:32:14,133 DEBUG [master/04e93f2b5fad:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/04e93f2b5fad:0, corePoolSize=2, maxPoolSize=2 2024-11-16T14:32:14,133 DEBUG [master/04e93f2b5fad:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/04e93f2b5fad:0, corePoolSize=1, maxPoolSize=1 2024-11-16T14:32:14,140 INFO [master/04e93f2b5fad:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731767564140 2024-11-16T14:32:14,141 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-16T14:32:14,142 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-16T14:32:14,142 INFO [master/04e93f2b5fad:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-16T14:32:14,142 DEBUG [RS:1;04e93f2b5fad:39921 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-16T14:32:14,142 DEBUG [RS:2;04e93f2b5fad:45119 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-16T14:32:14,142 DEBUG [RS:0;04e93f2b5fad:35241 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-16T14:32:14,143 INFO [master/04e93f2b5fad:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-16T14:32:14,147 INFO [master/04e93f2b5fad:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-16T14:32:14,147 INFO [master/04e93f2b5fad:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-16T14:32:14,148 INFO [master/04e93f2b5fad:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-16T14:32:14,148 INFO [master/04e93f2b5fad:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-16T14:32:14,149 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:32:14,149 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-16T14:32:14,149 INFO [master/04e93f2b5fad:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-16T14:32:14,154 INFO [master/04e93f2b5fad:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-16T14:32:14,155 INFO [master/04e93f2b5fad:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-16T14:32:14,156 INFO [master/04e93f2b5fad:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-16T14:32:14,160 INFO [master/04e93f2b5fad:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-16T14:32:14,161 INFO [master/04e93f2b5fad:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-16T14:32:14,165 DEBUG [master/04e93f2b5fad:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/04e93f2b5fad:0:becomeActiveMaster-HFileCleaner.large.0-1731767534162,5,FailOnTimeoutGroup] 2024-11-16T14:32:14,166 DEBUG [master/04e93f2b5fad:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/04e93f2b5fad:0:becomeActiveMaster-HFileCleaner.small.0-1731767534165,5,FailOnTimeoutGroup] 2024-11-16T14:32:14,166 INFO [master/04e93f2b5fad:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-16T14:32:14,166 INFO [master/04e93f2b5fad:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-16T14:32:14,193 INFO [master/04e93f2b5fad:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-16T14:32:14,193 INFO [master/04e93f2b5fad:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-16T14:32:14,195 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33005 is added to blk_1073741831_1007 (size=1321) 2024-11-16T14:32:14,195 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36193 is added to blk_1073741831_1007 (size=1321) 2024-11-16T14:32:14,196 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43279 is added to blk_1073741831_1007 (size=1321) 2024-11-16T14:32:14,198 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:43457/user/jenkins/test-data/0d13f5ec-97ce-91d4-7824-c5174956db56/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-16T14:32:14,199 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:43457/user/jenkins/test-data/0d13f5ec-97ce-91d4-7824-c5174956db56 2024-11-16T14:32:14,208 INFO [HMaster-EventLoopGroup-2-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42509, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-16T14:32:14,208 INFO [HMaster-EventLoopGroup-2-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52829, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.1 (auth:SIMPLE), service=RegionServerStatusService 2024-11-16T14:32:14,208 INFO [HMaster-EventLoopGroup-2-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49675, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.2 (auth:SIMPLE), service=RegionServerStatusService 2024-11-16T14:32:14,214 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34365 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 04e93f2b5fad,39921,1731767532970 2024-11-16T14:32:14,216 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34365 {}] master.ServerManager(517): Registering regionserver=04e93f2b5fad,39921,1731767532970 2024-11-16T14:32:14,221 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43279 is added to blk_1073741832_1008 (size=32) 2024-11-16T14:32:14,221 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33005 is added to blk_1073741832_1008 (size=32) 2024-11-16T14:32:14,223 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36193 is added to blk_1073741832_1008 (size=32) 2024-11-16T14:32:14,224 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T14:32:14,227 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-16T14:32:14,227 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34365 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 04e93f2b5fad,45119,1731767533018 2024-11-16T14:32:14,227 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34365 {}] master.ServerManager(517): Registering regionserver=04e93f2b5fad,45119,1731767533018 2024-11-16T14:32:14,231 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-16T14:32:14,231 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:32:14,231 DEBUG [RS:1;04e93f2b5fad:39921 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:43457/user/jenkins/test-data/0d13f5ec-97ce-91d4-7824-c5174956db56 2024-11-16T14:32:14,231 DEBUG [RS:2;04e93f2b5fad:45119 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:43457/user/jenkins/test-data/0d13f5ec-97ce-91d4-7824-c5174956db56 2024-11-16T14:32:14,232 DEBUG [RS:1;04e93f2b5fad:39921 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:43457 2024-11-16T14:32:14,232 DEBUG [RS:1;04e93f2b5fad:39921 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-16T14:32:14,232 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34365 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 04e93f2b5fad,35241,1731767532873 2024-11-16T14:32:14,232 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34365 {}] master.ServerManager(517): Registering regionserver=04e93f2b5fad,35241,1731767532873 2024-11-16T14:32:14,233 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-16T14:32:14,233 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-16T14:32:14,232 DEBUG [RS:2;04e93f2b5fad:45119 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:43457 2024-11-16T14:32:14,234 DEBUG [RS:2;04e93f2b5fad:45119 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-16T14:32:14,236 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-16T14:32:14,236 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:32:14,236 DEBUG [RS:0;04e93f2b5fad:35241 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:43457/user/jenkins/test-data/0d13f5ec-97ce-91d4-7824-c5174956db56 2024-11-16T14:32:14,237 DEBUG [RS:0;04e93f2b5fad:35241 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:43457 2024-11-16T14:32:14,237 DEBUG [RS:0;04e93f2b5fad:35241 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-16T14:32:14,238 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-16T14:32:14,238 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-16T14:32:14,240 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-16T14:32:14,240 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:32:14,242 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-16T14:32:14,242 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-16T14:32:14,244 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34365-0x101440f72da0000, quorum=127.0.0.1:57212, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-16T14:32:14,246 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-16T14:32:14,246 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:32:14,247 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-16T14:32:14,248 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-16T14:32:14,249 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43457/user/jenkins/test-data/0d13f5ec-97ce-91d4-7824-c5174956db56/data/hbase/meta/1588230740 2024-11-16T14:32:14,250 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43457/user/jenkins/test-data/0d13f5ec-97ce-91d4-7824-c5174956db56/data/hbase/meta/1588230740 2024-11-16T14:32:14,252 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-16T14:32:14,252 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-16T14:32:14,253 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-16T14:32:14,256 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-16T14:32:14,260 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43457/user/jenkins/test-data/0d13f5ec-97ce-91d4-7824-c5174956db56/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-16T14:32:14,261 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70119295, jitterRate=0.044858917593955994}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-16T14:32:14,264 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731767534224Initializing all the Stores at 1731767534226 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731767534226Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731767534226Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731767534226Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731767534227 (+1 ms)Cleaning up temporary data from old regions at 1731767534253 (+26 ms)Region opened successfully at 1731767534264 (+11 ms) 2024-11-16T14:32:14,265 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-16T14:32:14,265 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-16T14:32:14,265 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-16T14:32:14,265 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-16T14:32:14,265 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-16T14:32:14,266 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-16T14:32:14,266 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731767534264Disabling compacts and flushes for region at 1731767534264Disabling writes for close at 1731767534265 (+1 ms)Writing region close event to WAL at 1731767534266 (+1 ms)Closed at 1731767534266 2024-11-16T14:32:14,269 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-16T14:32:14,269 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-16T14:32:14,275 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-16T14:32:14,283 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-16T14:32:14,287 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-16T14:32:14,287 DEBUG [RS:1;04e93f2b5fad:39921 {}] zookeeper.ZKUtil(111): regionserver:39921-0x101440f72da0002, quorum=127.0.0.1:57212, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/04e93f2b5fad,39921,1731767532970 2024-11-16T14:32:14,287 WARN [RS:1;04e93f2b5fad:39921 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-16T14:32:14,287 INFO [RS:1;04e93f2b5fad:39921 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-16T14:32:14,287 DEBUG [RS:0;04e93f2b5fad:35241 {}] zookeeper.ZKUtil(111): regionserver:35241-0x101440f72da0001, quorum=127.0.0.1:57212, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/04e93f2b5fad,35241,1731767532873 2024-11-16T14:32:14,287 DEBUG [RS:2;04e93f2b5fad:45119 {}] zookeeper.ZKUtil(111): regionserver:45119-0x101440f72da0003, quorum=127.0.0.1:57212, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/04e93f2b5fad,45119,1731767533018 2024-11-16T14:32:14,287 DEBUG [RS:1;04e93f2b5fad:39921 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:43457/user/jenkins/test-data/0d13f5ec-97ce-91d4-7824-c5174956db56/WALs/04e93f2b5fad,39921,1731767532970 2024-11-16T14:32:14,287 WARN [RS:0;04e93f2b5fad:35241 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-16T14:32:14,287 WARN [RS:2;04e93f2b5fad:45119 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-16T14:32:14,288 INFO [RS:0;04e93f2b5fad:35241 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-16T14:32:14,288 INFO [RS:2;04e93f2b5fad:45119 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-16T14:32:14,288 DEBUG [RS:0;04e93f2b5fad:35241 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:43457/user/jenkins/test-data/0d13f5ec-97ce-91d4-7824-c5174956db56/WALs/04e93f2b5fad,35241,1731767532873 2024-11-16T14:32:14,288 DEBUG [RS:2;04e93f2b5fad:45119 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:43457/user/jenkins/test-data/0d13f5ec-97ce-91d4-7824-c5174956db56/WALs/04e93f2b5fad,45119,1731767533018 2024-11-16T14:32:14,289 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [04e93f2b5fad,39921,1731767532970] 2024-11-16T14:32:14,290 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [04e93f2b5fad,45119,1731767533018] 2024-11-16T14:32:14,290 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [04e93f2b5fad,35241,1731767532873] 2024-11-16T14:32:14,313 INFO [RS:1;04e93f2b5fad:39921 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-16T14:32:14,313 INFO [RS:0;04e93f2b5fad:35241 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-16T14:32:14,313 INFO [RS:2;04e93f2b5fad:45119 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-16T14:32:14,329 INFO [RS:0;04e93f2b5fad:35241 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-16T14:32:14,329 INFO [RS:1;04e93f2b5fad:39921 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-16T14:32:14,329 INFO [RS:2;04e93f2b5fad:45119 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-16T14:32:14,336 INFO [RS:0;04e93f2b5fad:35241 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-16T14:32:14,336 INFO [RS:1;04e93f2b5fad:39921 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-16T14:32:14,336 INFO [RS:2;04e93f2b5fad:45119 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-16T14:32:14,336 INFO [RS:0;04e93f2b5fad:35241 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-16T14:32:14,336 INFO [RS:2;04e93f2b5fad:45119 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-16T14:32:14,336 INFO [RS:1;04e93f2b5fad:39921 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-16T14:32:14,338 INFO [RS:2;04e93f2b5fad:45119 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-16T14:32:14,338 INFO [RS:1;04e93f2b5fad:39921 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-16T14:32:14,338 INFO [RS:0;04e93f2b5fad:35241 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-16T14:32:14,343 INFO [RS:2;04e93f2b5fad:45119 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-16T14:32:14,343 INFO [RS:0;04e93f2b5fad:35241 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-16T14:32:14,343 INFO [RS:1;04e93f2b5fad:39921 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-16T14:32:14,345 INFO [RS:2;04e93f2b5fad:45119 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-16T14:32:14,345 INFO [RS:0;04e93f2b5fad:35241 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-16T14:32:14,345 INFO [RS:1;04e93f2b5fad:39921 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-16T14:32:14,345 DEBUG [RS:0;04e93f2b5fad:35241 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/04e93f2b5fad:0, corePoolSize=1, maxPoolSize=1 2024-11-16T14:32:14,345 DEBUG [RS:2;04e93f2b5fad:45119 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/04e93f2b5fad:0, corePoolSize=1, maxPoolSize=1 2024-11-16T14:32:14,345 DEBUG [RS:1;04e93f2b5fad:39921 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/04e93f2b5fad:0, corePoolSize=1, maxPoolSize=1 2024-11-16T14:32:14,345 DEBUG [RS:2;04e93f2b5fad:45119 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/04e93f2b5fad:0, corePoolSize=1, maxPoolSize=1 2024-11-16T14:32:14,345 DEBUG [RS:0;04e93f2b5fad:35241 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/04e93f2b5fad:0, corePoolSize=1, maxPoolSize=1 2024-11-16T14:32:14,345 DEBUG [RS:1;04e93f2b5fad:39921 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/04e93f2b5fad:0, corePoolSize=1, maxPoolSize=1 2024-11-16T14:32:14,345 DEBUG [RS:2;04e93f2b5fad:45119 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/04e93f2b5fad:0, corePoolSize=1, maxPoolSize=1 2024-11-16T14:32:14,345 DEBUG [RS:0;04e93f2b5fad:35241 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/04e93f2b5fad:0, corePoolSize=1, maxPoolSize=1 2024-11-16T14:32:14,345 DEBUG [RS:1;04e93f2b5fad:39921 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/04e93f2b5fad:0, corePoolSize=1, maxPoolSize=1 2024-11-16T14:32:14,345 DEBUG [RS:2;04e93f2b5fad:45119 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/04e93f2b5fad:0, corePoolSize=1, maxPoolSize=1 2024-11-16T14:32:14,345 DEBUG [RS:2;04e93f2b5fad:45119 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/04e93f2b5fad:0, corePoolSize=1, maxPoolSize=1 2024-11-16T14:32:14,345 DEBUG [RS:0;04e93f2b5fad:35241 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/04e93f2b5fad:0, corePoolSize=1, maxPoolSize=1 2024-11-16T14:32:14,346 DEBUG [RS:1;04e93f2b5fad:39921 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/04e93f2b5fad:0, corePoolSize=1, maxPoolSize=1 2024-11-16T14:32:14,346 DEBUG [RS:2;04e93f2b5fad:45119 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/04e93f2b5fad:0, corePoolSize=2, maxPoolSize=2 2024-11-16T14:32:14,346 DEBUG [RS:0;04e93f2b5fad:35241 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/04e93f2b5fad:0, corePoolSize=1, maxPoolSize=1 2024-11-16T14:32:14,346 DEBUG [RS:1;04e93f2b5fad:39921 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/04e93f2b5fad:0, corePoolSize=1, maxPoolSize=1 2024-11-16T14:32:14,346 DEBUG [RS:2;04e93f2b5fad:45119 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/04e93f2b5fad:0, corePoolSize=1, maxPoolSize=1 2024-11-16T14:32:14,346 DEBUG [RS:0;04e93f2b5fad:35241 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/04e93f2b5fad:0, corePoolSize=2, maxPoolSize=2 2024-11-16T14:32:14,346 DEBUG [RS:2;04e93f2b5fad:45119 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/04e93f2b5fad:0, corePoolSize=1, maxPoolSize=1 2024-11-16T14:32:14,346 DEBUG [RS:1;04e93f2b5fad:39921 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/04e93f2b5fad:0, corePoolSize=2, maxPoolSize=2 2024-11-16T14:32:14,346 DEBUG [RS:0;04e93f2b5fad:35241 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/04e93f2b5fad:0, corePoolSize=1, maxPoolSize=1 2024-11-16T14:32:14,346 DEBUG [RS:2;04e93f2b5fad:45119 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/04e93f2b5fad:0, corePoolSize=1, maxPoolSize=1 2024-11-16T14:32:14,346 DEBUG [RS:1;04e93f2b5fad:39921 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/04e93f2b5fad:0, corePoolSize=1, maxPoolSize=1 2024-11-16T14:32:14,346 DEBUG [RS:0;04e93f2b5fad:35241 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/04e93f2b5fad:0, corePoolSize=1, maxPoolSize=1 2024-11-16T14:32:14,346 DEBUG [RS:2;04e93f2b5fad:45119 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/04e93f2b5fad:0, corePoolSize=1, maxPoolSize=1 2024-11-16T14:32:14,346 DEBUG [RS:1;04e93f2b5fad:39921 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/04e93f2b5fad:0, corePoolSize=1, maxPoolSize=1 2024-11-16T14:32:14,346 DEBUG [RS:0;04e93f2b5fad:35241 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/04e93f2b5fad:0, corePoolSize=1, maxPoolSize=1 2024-11-16T14:32:14,346 DEBUG [RS:2;04e93f2b5fad:45119 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/04e93f2b5fad:0, corePoolSize=1, maxPoolSize=1 2024-11-16T14:32:14,346 DEBUG [RS:1;04e93f2b5fad:39921 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/04e93f2b5fad:0, corePoolSize=1, maxPoolSize=1 2024-11-16T14:32:14,346 DEBUG [RS:0;04e93f2b5fad:35241 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/04e93f2b5fad:0, corePoolSize=1, maxPoolSize=1 2024-11-16T14:32:14,347 DEBUG [RS:2;04e93f2b5fad:45119 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/04e93f2b5fad:0, corePoolSize=1, maxPoolSize=1 2024-11-16T14:32:14,347 DEBUG [RS:1;04e93f2b5fad:39921 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/04e93f2b5fad:0, corePoolSize=1, maxPoolSize=1 2024-11-16T14:32:14,347 DEBUG [RS:0;04e93f2b5fad:35241 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/04e93f2b5fad:0, corePoolSize=1, maxPoolSize=1 2024-11-16T14:32:14,347 DEBUG [RS:2;04e93f2b5fad:45119 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/04e93f2b5fad:0, corePoolSize=3, maxPoolSize=3 2024-11-16T14:32:14,347 DEBUG [RS:1;04e93f2b5fad:39921 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/04e93f2b5fad:0, corePoolSize=1, maxPoolSize=1 2024-11-16T14:32:14,347 DEBUG [RS:0;04e93f2b5fad:35241 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/04e93f2b5fad:0, corePoolSize=1, maxPoolSize=1 2024-11-16T14:32:14,347 DEBUG [RS:2;04e93f2b5fad:45119 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0, corePoolSize=3, maxPoolSize=3 2024-11-16T14:32:14,347 DEBUG [RS:1;04e93f2b5fad:39921 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/04e93f2b5fad:0, corePoolSize=1, maxPoolSize=1 2024-11-16T14:32:14,347 DEBUG [RS:0;04e93f2b5fad:35241 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/04e93f2b5fad:0, corePoolSize=3, maxPoolSize=3 2024-11-16T14:32:14,347 DEBUG [RS:1;04e93f2b5fad:39921 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/04e93f2b5fad:0, corePoolSize=3, maxPoolSize=3 2024-11-16T14:32:14,347 DEBUG [RS:0;04e93f2b5fad:35241 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0, corePoolSize=3, maxPoolSize=3 2024-11-16T14:32:14,347 DEBUG [RS:1;04e93f2b5fad:39921 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0, corePoolSize=3, maxPoolSize=3 2024-11-16T14:32:14,351 INFO [RS:2;04e93f2b5fad:45119 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-16T14:32:14,351 INFO [RS:0;04e93f2b5fad:35241 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-16T14:32:14,351 INFO [RS:1;04e93f2b5fad:39921 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-16T14:32:14,351 INFO [RS:2;04e93f2b5fad:45119 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-16T14:32:14,351 INFO [RS:0;04e93f2b5fad:35241 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-16T14:32:14,351 INFO [RS:1;04e93f2b5fad:39921 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-16T14:32:14,351 INFO [RS:2;04e93f2b5fad:45119 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-16T14:32:14,351 INFO [RS:0;04e93f2b5fad:35241 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-16T14:32:14,352 INFO [RS:1;04e93f2b5fad:39921 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-16T14:32:14,352 INFO [RS:2;04e93f2b5fad:45119 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-16T14:32:14,352 INFO [RS:0;04e93f2b5fad:35241 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-16T14:32:14,352 INFO [RS:1;04e93f2b5fad:39921 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-16T14:32:14,352 INFO [RS:2;04e93f2b5fad:45119 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-16T14:32:14,352 INFO [RS:0;04e93f2b5fad:35241 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-16T14:32:14,352 INFO [RS:1;04e93f2b5fad:39921 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-16T14:32:14,352 INFO [RS:2;04e93f2b5fad:45119 {}] hbase.ChoreService(168): Chore ScheduledChore name=04e93f2b5fad,45119,1731767533018-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-16T14:32:14,352 INFO [RS:0;04e93f2b5fad:35241 {}] hbase.ChoreService(168): Chore ScheduledChore name=04e93f2b5fad,35241,1731767532873-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-16T14:32:14,352 INFO [RS:1;04e93f2b5fad:39921 {}] hbase.ChoreService(168): Chore ScheduledChore name=04e93f2b5fad,39921,1731767532970-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-16T14:32:14,371 INFO [RS:0;04e93f2b5fad:35241 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-16T14:32:14,373 INFO [RS:2;04e93f2b5fad:45119 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-16T14:32:14,373 INFO [RS:1;04e93f2b5fad:39921 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-16T14:32:14,373 INFO [RS:0;04e93f2b5fad:35241 {}] hbase.ChoreService(168): Chore ScheduledChore name=04e93f2b5fad,35241,1731767532873-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-16T14:32:14,373 INFO [RS:2;04e93f2b5fad:45119 {}] hbase.ChoreService(168): Chore ScheduledChore name=04e93f2b5fad,45119,1731767533018-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-16T14:32:14,373 INFO [RS:1;04e93f2b5fad:39921 {}] hbase.ChoreService(168): Chore ScheduledChore name=04e93f2b5fad,39921,1731767532970-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-16T14:32:14,374 INFO [RS:1;04e93f2b5fad:39921 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-16T14:32:14,374 INFO [RS:0;04e93f2b5fad:35241 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-16T14:32:14,374 INFO [RS:2;04e93f2b5fad:45119 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-16T14:32:14,374 INFO [RS:1;04e93f2b5fad:39921 {}] regionserver.Replication(171): 04e93f2b5fad,39921,1731767532970 started 2024-11-16T14:32:14,374 INFO [RS:0;04e93f2b5fad:35241 {}] regionserver.Replication(171): 04e93f2b5fad,35241,1731767532873 started 2024-11-16T14:32:14,374 INFO [RS:2;04e93f2b5fad:45119 {}] regionserver.Replication(171): 04e93f2b5fad,45119,1731767533018 started 2024-11-16T14:32:14,394 INFO [RS:1;04e93f2b5fad:39921 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-16T14:32:14,394 INFO [RS:1;04e93f2b5fad:39921 {}] regionserver.HRegionServer(1482): Serving as 04e93f2b5fad,39921,1731767532970, RpcServer on 04e93f2b5fad/172.17.0.2:39921, sessionid=0x101440f72da0002 2024-11-16T14:32:14,394 DEBUG [RS:1;04e93f2b5fad:39921 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-16T14:32:14,395 DEBUG [RS:1;04e93f2b5fad:39921 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 04e93f2b5fad,39921,1731767532970 2024-11-16T14:32:14,395 DEBUG [RS:1;04e93f2b5fad:39921 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '04e93f2b5fad,39921,1731767532970' 2024-11-16T14:32:14,395 DEBUG [RS:1;04e93f2b5fad:39921 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-16T14:32:14,396 DEBUG [RS:1;04e93f2b5fad:39921 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-16T14:32:14,396 INFO [RS:0;04e93f2b5fad:35241 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-16T14:32:14,396 INFO [RS:2;04e93f2b5fad:45119 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-16T14:32:14,396 INFO [RS:0;04e93f2b5fad:35241 {}] regionserver.HRegionServer(1482): Serving as 04e93f2b5fad,35241,1731767532873, RpcServer on 04e93f2b5fad/172.17.0.2:35241, sessionid=0x101440f72da0001 2024-11-16T14:32:14,397 INFO [RS:2;04e93f2b5fad:45119 {}] regionserver.HRegionServer(1482): Serving as 04e93f2b5fad,45119,1731767533018, RpcServer on 04e93f2b5fad/172.17.0.2:45119, sessionid=0x101440f72da0003 2024-11-16T14:32:14,397 DEBUG [RS:1;04e93f2b5fad:39921 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-16T14:32:14,397 DEBUG [RS:1;04e93f2b5fad:39921 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-16T14:32:14,397 DEBUG [RS:0;04e93f2b5fad:35241 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-16T14:32:14,397 DEBUG [RS:2;04e93f2b5fad:45119 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-16T14:32:14,397 DEBUG [RS:0;04e93f2b5fad:35241 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 04e93f2b5fad,35241,1731767532873 2024-11-16T14:32:14,397 DEBUG [RS:1;04e93f2b5fad:39921 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 04e93f2b5fad,39921,1731767532970 2024-11-16T14:32:14,397 DEBUG [RS:0;04e93f2b5fad:35241 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '04e93f2b5fad,35241,1731767532873' 2024-11-16T14:32:14,397 DEBUG [RS:2;04e93f2b5fad:45119 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 04e93f2b5fad,45119,1731767533018 2024-11-16T14:32:14,397 DEBUG [RS:1;04e93f2b5fad:39921 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '04e93f2b5fad,39921,1731767532970' 2024-11-16T14:32:14,397 DEBUG [RS:0;04e93f2b5fad:35241 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-16T14:32:14,397 DEBUG [RS:2;04e93f2b5fad:45119 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '04e93f2b5fad,45119,1731767533018' 2024-11-16T14:32:14,397 DEBUG [RS:1;04e93f2b5fad:39921 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-16T14:32:14,397 DEBUG [RS:2;04e93f2b5fad:45119 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-16T14:32:14,398 DEBUG [RS:1;04e93f2b5fad:39921 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-16T14:32:14,398 DEBUG [RS:0;04e93f2b5fad:35241 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-16T14:32:14,398 DEBUG [RS:2;04e93f2b5fad:45119 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-16T14:32:14,398 DEBUG [RS:1;04e93f2b5fad:39921 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-16T14:32:14,398 DEBUG [RS:0;04e93f2b5fad:35241 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-16T14:32:14,398 DEBUG [RS:2;04e93f2b5fad:45119 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-16T14:32:14,398 DEBUG [RS:0;04e93f2b5fad:35241 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-16T14:32:14,398 INFO [RS:1;04e93f2b5fad:39921 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-16T14:32:14,398 DEBUG [RS:2;04e93f2b5fad:45119 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-16T14:32:14,398 DEBUG [RS:0;04e93f2b5fad:35241 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 04e93f2b5fad,35241,1731767532873 2024-11-16T14:32:14,398 INFO [RS:1;04e93f2b5fad:39921 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-16T14:32:14,398 DEBUG [RS:2;04e93f2b5fad:45119 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 04e93f2b5fad,45119,1731767533018 2024-11-16T14:32:14,398 DEBUG [RS:0;04e93f2b5fad:35241 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '04e93f2b5fad,35241,1731767532873' 2024-11-16T14:32:14,398 DEBUG [RS:2;04e93f2b5fad:45119 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '04e93f2b5fad,45119,1731767533018' 2024-11-16T14:32:14,398 DEBUG [RS:0;04e93f2b5fad:35241 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-16T14:32:14,398 DEBUG [RS:2;04e93f2b5fad:45119 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-16T14:32:14,399 DEBUG [RS:2;04e93f2b5fad:45119 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-16T14:32:14,399 DEBUG [RS:0;04e93f2b5fad:35241 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-16T14:32:14,399 DEBUG [RS:2;04e93f2b5fad:45119 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-16T14:32:14,400 DEBUG [RS:0;04e93f2b5fad:35241 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-16T14:32:14,400 INFO [RS:2;04e93f2b5fad:45119 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-16T14:32:14,400 INFO [RS:0;04e93f2b5fad:35241 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-16T14:32:14,400 INFO [RS:2;04e93f2b5fad:45119 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-16T14:32:14,400 INFO [RS:0;04e93f2b5fad:35241 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-16T14:32:14,438 WARN [04e93f2b5fad:34365 {}] assignment.AssignmentManager(2443): No servers available; cannot place 1 unassigned regions. 2024-11-16T14:32:14,504 INFO [RS:0;04e93f2b5fad:35241 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-16T14:32:14,504 INFO [RS:1;04e93f2b5fad:39921 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-16T14:32:14,504 INFO [RS:2;04e93f2b5fad:45119 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-16T14:32:14,508 INFO [RS:2;04e93f2b5fad:45119 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=04e93f2b5fad%2C45119%2C1731767533018, suffix=, logDir=hdfs://localhost:43457/user/jenkins/test-data/0d13f5ec-97ce-91d4-7824-c5174956db56/WALs/04e93f2b5fad,45119,1731767533018, archiveDir=hdfs://localhost:43457/user/jenkins/test-data/0d13f5ec-97ce-91d4-7824-c5174956db56/oldWALs, maxLogs=32 2024-11-16T14:32:14,509 INFO [RS:1;04e93f2b5fad:39921 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=04e93f2b5fad%2C39921%2C1731767532970, suffix=, logDir=hdfs://localhost:43457/user/jenkins/test-data/0d13f5ec-97ce-91d4-7824-c5174956db56/WALs/04e93f2b5fad,39921,1731767532970, archiveDir=hdfs://localhost:43457/user/jenkins/test-data/0d13f5ec-97ce-91d4-7824-c5174956db56/oldWALs, maxLogs=32 2024-11-16T14:32:14,509 INFO [RS:0;04e93f2b5fad:35241 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=04e93f2b5fad%2C35241%2C1731767532873, suffix=, logDir=hdfs://localhost:43457/user/jenkins/test-data/0d13f5ec-97ce-91d4-7824-c5174956db56/WALs/04e93f2b5fad,35241,1731767532873, archiveDir=hdfs://localhost:43457/user/jenkins/test-data/0d13f5ec-97ce-91d4-7824-c5174956db56/oldWALs, maxLogs=32 2024-11-16T14:32:14,525 DEBUG [RS:2;04e93f2b5fad:45119 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/0d13f5ec-97ce-91d4-7824-c5174956db56/WALs/04e93f2b5fad,45119,1731767533018/04e93f2b5fad%2C45119%2C1731767533018.1731767534512, exclude list is [], retry=0 2024-11-16T14:32:14,526 DEBUG [RS:0;04e93f2b5fad:35241 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/0d13f5ec-97ce-91d4-7824-c5174956db56/WALs/04e93f2b5fad,35241,1731767532873/04e93f2b5fad%2C35241%2C1731767532873.1731767534512, exclude list is [], retry=0 2024-11-16T14:32:14,527 DEBUG [RS:1;04e93f2b5fad:39921 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/0d13f5ec-97ce-91d4-7824-c5174956db56/WALs/04e93f2b5fad,39921,1731767532970/04e93f2b5fad%2C39921%2C1731767532970.1731767534512, exclude list is [], retry=0 2024-11-16T14:32:14,529 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33005,DS-6a1dd528-c0be-4580-a53e-7113879f7f97,DISK] 2024-11-16T14:32:14,529 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43279,DS-17ab9554-4509-49b3-918b-178692b9a473,DISK] 2024-11-16T14:32:14,530 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36193,DS-a30e140c-8532-4096-8cae-c8d713a3d4e3,DISK] 2024-11-16T14:32:14,530 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43279,DS-17ab9554-4509-49b3-918b-178692b9a473,DISK] 2024-11-16T14:32:14,530 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36193,DS-a30e140c-8532-4096-8cae-c8d713a3d4e3,DISK] 2024-11-16T14:32:14,530 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33005,DS-6a1dd528-c0be-4580-a53e-7113879f7f97,DISK] 2024-11-16T14:32:14,531 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36193,DS-a30e140c-8532-4096-8cae-c8d713a3d4e3,DISK] 2024-11-16T14:32:14,531 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33005,DS-6a1dd528-c0be-4580-a53e-7113879f7f97,DISK] 2024-11-16T14:32:14,531 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43279,DS-17ab9554-4509-49b3-918b-178692b9a473,DISK] 2024-11-16T14:32:14,538 INFO [RS:0;04e93f2b5fad:35241 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/0d13f5ec-97ce-91d4-7824-c5174956db56/WALs/04e93f2b5fad,35241,1731767532873/04e93f2b5fad%2C35241%2C1731767532873.1731767534512 2024-11-16T14:32:14,539 INFO [RS:2;04e93f2b5fad:45119 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/0d13f5ec-97ce-91d4-7824-c5174956db56/WALs/04e93f2b5fad,45119,1731767533018/04e93f2b5fad%2C45119%2C1731767533018.1731767534512 2024-11-16T14:32:14,539 DEBUG [RS:0;04e93f2b5fad:35241 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35677:35677),(127.0.0.1/127.0.0.1:41179:41179),(127.0.0.1/127.0.0.1:36015:36015)] 2024-11-16T14:32:14,539 DEBUG [RS:2;04e93f2b5fad:45119 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41179:41179),(127.0.0.1/127.0.0.1:36015:36015),(127.0.0.1/127.0.0.1:35677:35677)] 2024-11-16T14:32:14,539 INFO [RS:1;04e93f2b5fad:39921 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/0d13f5ec-97ce-91d4-7824-c5174956db56/WALs/04e93f2b5fad,39921,1731767532970/04e93f2b5fad%2C39921%2C1731767532970.1731767534512 2024-11-16T14:32:14,540 DEBUG [RS:1;04e93f2b5fad:39921 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41179:41179),(127.0.0.1/127.0.0.1:36015:36015),(127.0.0.1/127.0.0.1:35677:35677)] 2024-11-16T14:32:14,691 DEBUG [04e93f2b5fad:34365 {}] assignment.AssignmentManager(2464): Processing assignQueue; systemServersCount=3, allServersCount=3 2024-11-16T14:32:14,702 DEBUG [04e93f2b5fad:34365 {}] balancer.BalancerClusterState(204): Hosts are {04e93f2b5fad=0} racks are {/default-rack=0} 2024-11-16T14:32:14,707 DEBUG [04e93f2b5fad:34365 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-16T14:32:14,707 DEBUG [04e93f2b5fad:34365 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-16T14:32:14,708 DEBUG [04e93f2b5fad:34365 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-11-16T14:32:14,708 DEBUG [04e93f2b5fad:34365 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-16T14:32:14,708 DEBUG [04e93f2b5fad:34365 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-16T14:32:14,708 DEBUG [04e93f2b5fad:34365 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-11-16T14:32:14,708 INFO [04e93f2b5fad:34365 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-16T14:32:14,708 INFO [04e93f2b5fad:34365 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-16T14:32:14,708 INFO [04e93f2b5fad:34365 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-11-16T14:32:14,708 DEBUG [04e93f2b5fad:34365 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-16T14:32:14,714 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=04e93f2b5fad,35241,1731767532873 2024-11-16T14:32:14,719 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 04e93f2b5fad,35241,1731767532873, state=OPENING 2024-11-16T14:32:14,781 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-16T14:32:14,792 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35241-0x101440f72da0001, quorum=127.0.0.1:57212, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T14:32:14,792 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39921-0x101440f72da0002, quorum=127.0.0.1:57212, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T14:32:14,792 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45119-0x101440f72da0003, quorum=127.0.0.1:57212, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T14:32:14,792 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34365-0x101440f72da0000, quorum=127.0.0.1:57212, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T14:32:14,794 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T14:32:14,794 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T14:32:14,794 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T14:32:14,794 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T14:32:14,797 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-16T14:32:14,799 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=04e93f2b5fad,35241,1731767532873}] 2024-11-16T14:32:14,985 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-16T14:32:14,987 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40911, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-16T14:32:14,998 INFO [RS_OPEN_META-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-16T14:32:14,998 INFO [RS_OPEN_META-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-16T14:32:14,999 INFO [RS_OPEN_META-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-16T14:32:15,002 INFO [RS_OPEN_META-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=04e93f2b5fad%2C35241%2C1731767532873.meta, suffix=.meta, logDir=hdfs://localhost:43457/user/jenkins/test-data/0d13f5ec-97ce-91d4-7824-c5174956db56/WALs/04e93f2b5fad,35241,1731767532873, archiveDir=hdfs://localhost:43457/user/jenkins/test-data/0d13f5ec-97ce-91d4-7824-c5174956db56/oldWALs, maxLogs=32 2024-11-16T14:32:15,020 DEBUG [RS_OPEN_META-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/0d13f5ec-97ce-91d4-7824-c5174956db56/WALs/04e93f2b5fad,35241,1731767532873/04e93f2b5fad%2C35241%2C1731767532873.meta.1731767535004.meta, exclude list is [], retry=0 2024-11-16T14:32:15,024 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33005,DS-6a1dd528-c0be-4580-a53e-7113879f7f97,DISK] 2024-11-16T14:32:15,024 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36193,DS-a30e140c-8532-4096-8cae-c8d713a3d4e3,DISK] 2024-11-16T14:32:15,024 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43279,DS-17ab9554-4509-49b3-918b-178692b9a473,DISK] 2024-11-16T14:32:15,027 INFO [RS_OPEN_META-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/0d13f5ec-97ce-91d4-7824-c5174956db56/WALs/04e93f2b5fad,35241,1731767532873/04e93f2b5fad%2C35241%2C1731767532873.meta.1731767535004.meta 2024-11-16T14:32:15,027 DEBUG [RS_OPEN_META-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41179:41179),(127.0.0.1/127.0.0.1:35677:35677),(127.0.0.1/127.0.0.1:36015:36015)] 2024-11-16T14:32:15,027 DEBUG [RS_OPEN_META-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-16T14:32:15,029 DEBUG [RS_OPEN_META-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-16T14:32:15,031 DEBUG [RS_OPEN_META-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-16T14:32:15,035 INFO [RS_OPEN_META-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-16T14:32:15,039 DEBUG [RS_OPEN_META-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-16T14:32:15,039 DEBUG [RS_OPEN_META-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T14:32:15,039 DEBUG [RS_OPEN_META-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-16T14:32:15,039 DEBUG [RS_OPEN_META-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-16T14:32:15,042 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-16T14:32:15,044 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-16T14:32:15,044 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:32:15,045 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-16T14:32:15,045 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-16T14:32:15,047 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-16T14:32:15,047 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:32:15,047 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-16T14:32:15,048 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-16T14:32:15,049 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-16T14:32:15,049 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:32:15,049 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-16T14:32:15,049 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-16T14:32:15,051 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-16T14:32:15,051 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:32:15,051 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-16T14:32:15,052 DEBUG [RS_OPEN_META-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-16T14:32:15,053 DEBUG [RS_OPEN_META-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43457/user/jenkins/test-data/0d13f5ec-97ce-91d4-7824-c5174956db56/data/hbase/meta/1588230740 2024-11-16T14:32:15,056 DEBUG [RS_OPEN_META-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43457/user/jenkins/test-data/0d13f5ec-97ce-91d4-7824-c5174956db56/data/hbase/meta/1588230740 2024-11-16T14:32:15,058 DEBUG [RS_OPEN_META-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-16T14:32:15,058 DEBUG [RS_OPEN_META-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-16T14:32:15,059 DEBUG [RS_OPEN_META-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-16T14:32:15,062 DEBUG [RS_OPEN_META-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-16T14:32:15,064 INFO [RS_OPEN_META-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73923269, jitterRate=0.10154254734516144}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-16T14:32:15,064 DEBUG [RS_OPEN_META-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-16T14:32:15,066 DEBUG [RS_OPEN_META-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731767535040Writing region info on filesystem at 1731767535040Initializing all the Stores at 1731767535042 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731767535042Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731767535042Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731767535042Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731767535042Cleaning up temporary data from old regions at 1731767535058 (+16 ms)Running coprocessor post-open hooks at 1731767535064 (+6 ms)Region opened successfully at 1731767535066 (+2 ms) 2024-11-16T14:32:15,072 INFO [RS_OPEN_META-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731767534973 2024-11-16T14:32:15,088 DEBUG [RS_OPEN_META-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-16T14:32:15,089 INFO [RS_OPEN_META-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-16T14:32:15,090 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=04e93f2b5fad,35241,1731767532873 2024-11-16T14:32:15,092 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 04e93f2b5fad,35241,1731767532873, state=OPEN 2024-11-16T14:32:15,102 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35241-0x101440f72da0001, quorum=127.0.0.1:57212, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-16T14:32:15,102 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45119-0x101440f72da0003, quorum=127.0.0.1:57212, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-16T14:32:15,102 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34365-0x101440f72da0000, quorum=127.0.0.1:57212, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-16T14:32:15,102 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39921-0x101440f72da0002, quorum=127.0.0.1:57212, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-16T14:32:15,103 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T14:32:15,103 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T14:32:15,103 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T14:32:15,103 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T14:32:15,103 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=04e93f2b5fad,35241,1731767532873 2024-11-16T14:32:15,108 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-16T14:32:15,109 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=04e93f2b5fad,35241,1731767532873 in 304 msec 2024-11-16T14:32:15,118 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-16T14:32:15,118 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 835 msec 2024-11-16T14:32:15,120 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-16T14:32:15,120 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-16T14:32:15,137 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T14:32:15,138 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=04e93f2b5fad,35241,1731767532873, seqNum=-1] 2024-11-16T14:32:15,157 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T14:32:15,159 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45173, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T14:32:15,177 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.1400 sec 2024-11-16T14:32:15,177 INFO [master/04e93f2b5fad:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731767535177, completionTime=-1 2024-11-16T14:32:15,180 INFO [master/04e93f2b5fad:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=3; waited=0ms, expected min=3 server(s), max=3 server(s), master is running 2024-11-16T14:32:15,180 DEBUG [master/04e93f2b5fad:0:becomeActiveMaster {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-16T14:32:15,223 INFO [master/04e93f2b5fad:0:becomeActiveMaster {}] assignment.AssignmentManager(1768): Number of RegionServers=3 2024-11-16T14:32:15,223 INFO [master/04e93f2b5fad:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731767595223 2024-11-16T14:32:15,223 INFO [master/04e93f2b5fad:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731767655223 2024-11-16T14:32:15,223 INFO [master/04e93f2b5fad:0:becomeActiveMaster {}] assignment.AssignmentManager(1775): Joined the cluster in 43 msec 2024-11-16T14:32:15,225 DEBUG [master/04e93f2b5fad:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-11-16T14:32:15,234 INFO [master/04e93f2b5fad:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=04e93f2b5fad,34365,1731767532092-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-16T14:32:15,235 INFO [master/04e93f2b5fad:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=04e93f2b5fad,34365,1731767532092-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-16T14:32:15,235 INFO [master/04e93f2b5fad:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=04e93f2b5fad,34365,1731767532092-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-16T14:32:15,237 INFO [master/04e93f2b5fad:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-04e93f2b5fad:34365, period=300000, unit=MILLISECONDS is enabled. 2024-11-16T14:32:15,237 INFO [master/04e93f2b5fad:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-16T14:32:15,238 INFO [master/04e93f2b5fad:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-16T14:32:15,245 DEBUG [master/04e93f2b5fad:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-16T14:32:15,273 INFO [master/04e93f2b5fad:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 2.142sec 2024-11-16T14:32:15,274 INFO [master/04e93f2b5fad:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-16T14:32:15,276 INFO [master/04e93f2b5fad:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-16T14:32:15,277 INFO [master/04e93f2b5fad:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-16T14:32:15,277 INFO [master/04e93f2b5fad:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-16T14:32:15,277 INFO [master/04e93f2b5fad:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-16T14:32:15,278 INFO [master/04e93f2b5fad:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=04e93f2b5fad,34365,1731767532092-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-16T14:32:15,279 INFO [master/04e93f2b5fad:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=04e93f2b5fad,34365,1731767532092-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-16T14:32:15,283 DEBUG [master/04e93f2b5fad:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-16T14:32:15,284 INFO [master/04e93f2b5fad:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-16T14:32:15,285 INFO [master/04e93f2b5fad:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=04e93f2b5fad,34365,1731767532092-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-16T14:32:15,371 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@ddf6023, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:32:15,372 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 04e93f2b5fad,34365,-1 for getting cluster id 2024-11-16T14:32:15,374 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T14:32:15,382 DEBUG [HMaster-EventLoopGroup-2-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6ead4d3f-84d5-440b-9e8d-d5bc0db5a8c1' 2024-11-16T14:32:15,384 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T14:32:15,384 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6ead4d3f-84d5-440b-9e8d-d5bc0db5a8c1" 2024-11-16T14:32:15,384 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@54fbacc0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:32:15,385 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [04e93f2b5fad,34365,-1] 2024-11-16T14:32:15,387 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T14:32:15,389 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:32:15,390 INFO [HMaster-EventLoopGroup-2-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40952, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T14:32:15,392 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@65c83e31, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:32:15,393 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T14:32:15,399 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=04e93f2b5fad,35241,1731767532873, seqNum=-1] 2024-11-16T14:32:15,400 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T14:32:15,402 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34404, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T14:32:15,419 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=04e93f2b5fad,34365,1731767532092 2024-11-16T14:32:15,419 INFO [Time-limited test {}] wal.AbstractTestWALReplay(147): hbase.rootdir=hdfs://localhost:43457/hbase 2024-11-16T14:32:15,432 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayValueCompression#testNameConflictWhenSplit0 Thread=365, OpenFileDescriptor=611, MaxFileDescriptor=1048576, SystemLoadAverage=170, ProcessCount=11, AvailableMemoryMB=4899 2024-11-16T14:32:15,449 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-16T14:32:15,452 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-16T14:32:15,453 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-16T14:32:15,457 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-73959761, suffix=, logDir=hdfs://localhost:43457/hbase/WALs/hregion-73959761, archiveDir=hdfs://localhost:43457/hbase/oldWALs, maxLogs=32 2024-11-16T14:32:15,471 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-73959761/hregion-73959761.1731767535459, exclude list is [], retry=0 2024-11-16T14:32:15,475 DEBUG [AsyncFSWAL-8-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33005,DS-6a1dd528-c0be-4580-a53e-7113879f7f97,DISK] 2024-11-16T14:32:15,476 DEBUG [AsyncFSWAL-8-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43279,DS-17ab9554-4509-49b3-918b-178692b9a473,DISK] 2024-11-16T14:32:15,476 DEBUG [AsyncFSWAL-8-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36193,DS-a30e140c-8532-4096-8cae-c8d713a3d4e3,DISK] 2024-11-16T14:32:15,479 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-73959761/hregion-73959761.1731767535459 2024-11-16T14:32:15,480 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41179:41179),(127.0.0.1/127.0.0.1:36015:36015),(127.0.0.1/127.0.0.1:35677:35677)] 2024-11-16T14:32:15,480 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 01ff7f0ec04a08ce61c3eae0b0e1096b, NAME => 'testReplayEditsWrittenIntoWAL,,1731767535449.01ff7f0ec04a08ce61c3eae0b0e1096b.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenIntoWAL', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43457/hbase 2024-11-16T14:32:15,492 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33005 is added to blk_1073741838_1014 (size=64) 2024-11-16T14:32:15,492 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43279 is added to blk_1073741838_1014 (size=64) 2024-11-16T14:32:15,492 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36193 is added to blk_1073741838_1014 (size=64) 2024-11-16T14:32:15,493 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1731767535449.01ff7f0ec04a08ce61c3eae0b0e1096b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T14:32:15,496 INFO [StoreOpener-01ff7f0ec04a08ce61c3eae0b0e1096b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 01ff7f0ec04a08ce61c3eae0b0e1096b 2024-11-16T14:32:15,498 INFO [StoreOpener-01ff7f0ec04a08ce61c3eae0b0e1096b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 01ff7f0ec04a08ce61c3eae0b0e1096b columnFamilyName a 2024-11-16T14:32:15,498 DEBUG [StoreOpener-01ff7f0ec04a08ce61c3eae0b0e1096b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:32:15,499 INFO [StoreOpener-01ff7f0ec04a08ce61c3eae0b0e1096b-1 {}] regionserver.HStore(327): Store=01ff7f0ec04a08ce61c3eae0b0e1096b/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T14:32:15,499 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 01ff7f0ec04a08ce61c3eae0b0e1096b 2024-11-16T14:32:15,500 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43457/hbase/data/default/testReplayEditsWrittenIntoWAL/01ff7f0ec04a08ce61c3eae0b0e1096b 2024-11-16T14:32:15,501 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43457/hbase/data/default/testReplayEditsWrittenIntoWAL/01ff7f0ec04a08ce61c3eae0b0e1096b 2024-11-16T14:32:15,502 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 01ff7f0ec04a08ce61c3eae0b0e1096b 2024-11-16T14:32:15,502 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 01ff7f0ec04a08ce61c3eae0b0e1096b 2024-11-16T14:32:15,505 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 01ff7f0ec04a08ce61c3eae0b0e1096b 2024-11-16T14:32:15,508 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43457/hbase/data/default/testReplayEditsWrittenIntoWAL/01ff7f0ec04a08ce61c3eae0b0e1096b/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-16T14:32:15,509 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 01ff7f0ec04a08ce61c3eae0b0e1096b; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74264656, jitterRate=0.10662961006164551}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-16T14:32:15,510 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 01ff7f0ec04a08ce61c3eae0b0e1096b: Writing region info on filesystem at 1731767535493Initializing all the Stores at 1731767535495 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731767535495Cleaning up temporary data from old regions at 1731767535502 (+7 ms)Region opened successfully at 1731767535509 (+7 ms) 2024-11-16T14:32:15,510 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 01ff7f0ec04a08ce61c3eae0b0e1096b, disabling compactions & flushes 2024-11-16T14:32:15,510 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1731767535449.01ff7f0ec04a08ce61c3eae0b0e1096b. 2024-11-16T14:32:15,510 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1731767535449.01ff7f0ec04a08ce61c3eae0b0e1096b. 2024-11-16T14:32:15,510 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1731767535449.01ff7f0ec04a08ce61c3eae0b0e1096b. after waiting 0 ms 2024-11-16T14:32:15,510 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1731767535449.01ff7f0ec04a08ce61c3eae0b0e1096b. 2024-11-16T14:32:15,510 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1731767535449.01ff7f0ec04a08ce61c3eae0b0e1096b. 2024-11-16T14:32:15,510 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 01ff7f0ec04a08ce61c3eae0b0e1096b: Waiting for close lock at 1731767535510Disabling compacts and flushes for region at 1731767535510Disabling writes for close at 1731767535510Writing region close event to WAL at 1731767535510Closed at 1731767535510 2024-11-16T14:32:15,515 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43279 is added to blk_1073741837_1013 (size=95) 2024-11-16T14:32:15,515 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33005 is added to blk_1073741837_1013 (size=95) 2024-11-16T14:32:15,515 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36193 is added to blk_1073741837_1013 (size=95) 2024-11-16T14:32:15,521 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-16T14:32:15,521 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-73959761:(num 1731767535459) 2024-11-16T14:32:15,523 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-11-16T14:32:15,540 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43279 is added to blk_1073741839_1015 (size=320) 2024-11-16T14:32:15,540 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36193 is added to blk_1073741839_1015 (size=320) 2024-11-16T14:32:15,541 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33005 is added to blk_1073741839_1015 (size=320) 2024-11-16T14:32:15,542 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-11-16T14:32:15,553 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36193 is added to blk_1073741840_1016 (size=253) 2024-11-16T14:32:15,554 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33005 is added to blk_1073741840_1016 (size=253) 2024-11-16T14:32:15,554 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43279 is added to blk_1073741840_1016 (size=253) 2024-11-16T14:32:15,574 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:43457/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731767535448/wal-1, size=320 (320bytes) 2024-11-16T14:32:15,574 DEBUG [Time-limited test {}] util.RecoverLeaseFSUtils(47): Initialize RecoverLeaseFSUtils 2024-11-16T14:32:15,574 DEBUG [Time-limited test {}] util.RecoverLeaseFSUtils(59): set recoverLeaseMethod to org.apache.hadoop.fs.LeaseRecoverable.recoverLease() 2024-11-16T14:32:15,574 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:43457/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731767535448/wal-1 2024-11-16T14:32:15,579 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:43457/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731767535448/wal-1 after 3ms 2024-11-16T14:32:15,585 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:43457/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731767535448/wal-1: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-16T14:32:15,585 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:43457/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731767535448/wal-1 took 13ms 2024-11-16T14:32:15,595 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:43457/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731767535448/wal-1 so closing down 2024-11-16T14:32:15,595 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-16T14:32:15,598 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal-1.temp 2024-11-16T14:32:15,600 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:43457/hbase/data/default/testReplayEditsWrittenIntoWAL/01ff7f0ec04a08ce61c3eae0b0e1096b/recovered.edits/0000000000000000001-wal-1.temp 2024-11-16T14:32:15,601 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-16T14:32:15,610 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33005 is added to blk_1073741841_1017 (size=320) 2024-11-16T14:32:15,610 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36193 is added to blk_1073741841_1017 (size=320) 2024-11-16T14:32:15,611 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43279 is added to blk_1073741841_1017 (size=320) 2024-11-16T14:32:15,613 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:43457/hbase/data/default/testReplayEditsWrittenIntoWAL/01ff7f0ec04a08ce61c3eae0b0e1096b/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) 2024-11-16T14:32:15,616 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:43457/hbase/data/default/testReplayEditsWrittenIntoWAL/01ff7f0ec04a08ce61c3eae0b0e1096b/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:43457/hbase/data/default/testReplayEditsWrittenIntoWAL/01ff7f0ec04a08ce61c3eae0b0e1096b/recovered.edits/0000000000000000002 2024-11-16T14:32:15,620 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 2 edits across 1 Regions in 30 ms; skipped=0; WAL=hdfs://localhost:43457/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731767535448/wal-1, size=320, length=320, corrupted=false, cancelled=false 2024-11-16T14:32:15,620 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:43457/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731767535448/wal-1, journal: Splitting hdfs://localhost:43457/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731767535448/wal-1, size=320 (320bytes) at 1731767535574Finishing writing output for hdfs://localhost:43457/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731767535448/wal-1 so closing down at 1731767535595 (+21 ms)Creating recovered edits writer path=hdfs://localhost:43457/hbase/data/default/testReplayEditsWrittenIntoWAL/01ff7f0ec04a08ce61c3eae0b0e1096b/recovered.edits/0000000000000000001-wal-1.temp at 1731767535600 (+5 ms)3 split writer threads finished at 1731767535601 (+1 ms)Closed recovered edits writer path=hdfs://localhost:43457/hbase/data/default/testReplayEditsWrittenIntoWAL/01ff7f0ec04a08ce61c3eae0b0e1096b/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) at 1731767535613 (+12 ms)Rename recovered edits hdfs://localhost:43457/hbase/data/default/testReplayEditsWrittenIntoWAL/01ff7f0ec04a08ce61c3eae0b0e1096b/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:43457/hbase/data/default/testReplayEditsWrittenIntoWAL/01ff7f0ec04a08ce61c3eae0b0e1096b/recovered.edits/0000000000000000002 at 1731767535616 (+3 ms)Processed 2 edits across 1 Regions in 30 ms; skipped=0; WAL=hdfs://localhost:43457/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731767535448/wal-1, size=320, length=320, corrupted=false, cancelled=false at 1731767535620 (+4 ms) 2024-11-16T14:32:15,634 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:43457/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731767535448/wal-2, size=253 (253bytes) 2024-11-16T14:32:15,634 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:43457/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731767535448/wal-2 2024-11-16T14:32:15,635 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:43457/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731767535448/wal-2 after 0ms 2024-11-16T14:32:15,638 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:43457/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731767535448/wal-2: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-16T14:32:15,638 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:43457/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731767535448/wal-2 took 5ms 2024-11-16T14:32:15,641 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:43457/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731767535448/wal-2 so closing down 2024-11-16T14:32:15,641 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-16T14:32:15,644 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000002-wal-2.temp 2024-11-16T14:32:15,645 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:43457/hbase/data/default/testReplayEditsWrittenIntoWAL/01ff7f0ec04a08ce61c3eae0b0e1096b/recovered.edits/0000000000000000002-wal-2.temp 2024-11-16T14:32:15,646 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-16T14:32:15,653 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36193 is added to blk_1073741842_1018 (size=253) 2024-11-16T14:32:15,653 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43279 is added to blk_1073741842_1018 (size=253) 2024-11-16T14:32:15,654 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33005 is added to blk_1073741842_1018 (size=253) 2024-11-16T14:32:15,655 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:43457/hbase/data/default/testReplayEditsWrittenIntoWAL/01ff7f0ec04a08ce61c3eae0b0e1096b/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-11-16T14:32:15,659 DEBUG [split-log-closeStream-pool-0 {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:43457/hbase/data/default/testReplayEditsWrittenIntoWAL/01ff7f0ec04a08ce61c3eae0b0e1096b/recovered.edits/0000000000000000002: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-16T14:32:15,662 WARN [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(184): Found existing old edits file and we have less entries. Deleting hdfs://localhost:43457/hbase/data/default/testReplayEditsWrittenIntoWAL/01ff7f0ec04a08ce61c3eae0b0e1096b/recovered.edits/0000000000000000002-wal-2.temp, length=253 2024-11-16T14:32:15,664 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 1 edits across 1 Regions in 25 ms; skipped=0; WAL=hdfs://localhost:43457/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731767535448/wal-2, size=253, length=253, corrupted=false, cancelled=false 2024-11-16T14:32:15,664 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:43457/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731767535448/wal-2, journal: Splitting hdfs://localhost:43457/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731767535448/wal-2, size=253 (253bytes) at 1731767535634Finishing writing output for hdfs://localhost:43457/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731767535448/wal-2 so closing down at 1731767535641 (+7 ms)Creating recovered edits writer path=hdfs://localhost:43457/hbase/data/default/testReplayEditsWrittenIntoWAL/01ff7f0ec04a08ce61c3eae0b0e1096b/recovered.edits/0000000000000000002-wal-2.temp at 1731767535645 (+4 ms)3 split writer threads finished at 1731767535646 (+1 ms)Closed recovered edits writer path=hdfs://localhost:43457/hbase/data/default/testReplayEditsWrittenIntoWAL/01ff7f0ec04a08ce61c3eae0b0e1096b/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1731767535655 (+9 ms)Processed 1 edits across 1 Regions in 25 ms; skipped=0; WAL=hdfs://localhost:43457/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731767535448/wal-2, size=253, length=253, corrupted=false, cancelled=false at 1731767535664 (+9 ms) 2024-11-16T14:32:15,664 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-16T14:32:15,667 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:43457/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731767535448, archiveDir=hdfs://localhost:43457/hbase/oldWALs, maxLogs=32 2024-11-16T14:32:15,680 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testnameconflictwhensplit0-manual,16010,1731767535448/wal.1731767535668, exclude list is [], retry=0 2024-11-16T14:32:15,684 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36193,DS-a30e140c-8532-4096-8cae-c8d713a3d4e3,DISK] 2024-11-16T14:32:15,684 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33005,DS-6a1dd528-c0be-4580-a53e-7113879f7f97,DISK] 2024-11-16T14:32:15,685 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43279,DS-17ab9554-4509-49b3-918b-178692b9a473,DISK] 2024-11-16T14:32:15,687 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testnameconflictwhensplit0-manual,16010,1731767535448/wal.1731767535668 2024-11-16T14:32:15,687 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35677:35677),(127.0.0.1/127.0.0.1:41179:41179),(127.0.0.1/127.0.0.1:36015:36015)] 2024-11-16T14:32:15,687 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 01ff7f0ec04a08ce61c3eae0b0e1096b, NAME => 'testReplayEditsWrittenIntoWAL,,1731767535449.01ff7f0ec04a08ce61c3eae0b0e1096b.', STARTKEY => '', ENDKEY => ''} 2024-11-16T14:32:15,688 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1731767535449.01ff7f0ec04a08ce61c3eae0b0e1096b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T14:32:15,688 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 01ff7f0ec04a08ce61c3eae0b0e1096b 2024-11-16T14:32:15,688 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 01ff7f0ec04a08ce61c3eae0b0e1096b 2024-11-16T14:32:15,690 INFO [StoreOpener-01ff7f0ec04a08ce61c3eae0b0e1096b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 01ff7f0ec04a08ce61c3eae0b0e1096b 2024-11-16T14:32:15,691 INFO [StoreOpener-01ff7f0ec04a08ce61c3eae0b0e1096b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 01ff7f0ec04a08ce61c3eae0b0e1096b columnFamilyName a 2024-11-16T14:32:15,691 DEBUG [StoreOpener-01ff7f0ec04a08ce61c3eae0b0e1096b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:32:15,692 INFO [StoreOpener-01ff7f0ec04a08ce61c3eae0b0e1096b-1 {}] regionserver.HStore(327): Store=01ff7f0ec04a08ce61c3eae0b0e1096b/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T14:32:15,693 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 01ff7f0ec04a08ce61c3eae0b0e1096b 2024-11-16T14:32:15,694 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43457/hbase/data/default/testReplayEditsWrittenIntoWAL/01ff7f0ec04a08ce61c3eae0b0e1096b 2024-11-16T14:32:15,697 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:43457/hbase/data/default/testReplayEditsWrittenIntoWAL/01ff7f0ec04a08ce61c3eae0b0e1096b 2024-11-16T14:32:15,698 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:43457/hbase/data/default/testReplayEditsWrittenIntoWAL/01ff7f0ec04a08ce61c3eae0b0e1096b/recovered.edits/0000000000000000002 2024-11-16T14:32:15,703 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:43457/hbase/data/default/testReplayEditsWrittenIntoWAL/01ff7f0ec04a08ce61c3eae0b0e1096b/recovered.edits/0000000000000000002: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-16T14:32:15,709 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 2, skipped 0, firstSequenceIdInLog=1, maxSequenceIdInLog=2, path=hdfs://localhost:43457/hbase/data/default/testReplayEditsWrittenIntoWAL/01ff7f0ec04a08ce61c3eae0b0e1096b/recovered.edits/0000000000000000002 2024-11-16T14:32:15,712 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 01ff7f0ec04a08ce61c3eae0b0e1096b 1/1 column families, dataSize=108 B heapSize=512 B 2024-11-16T14:32:15,763 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43457/hbase/data/default/testReplayEditsWrittenIntoWAL/01ff7f0ec04a08ce61c3eae0b0e1096b/.tmp/a/efdfc40587474910bdef33bdbccf2664 is 58, key is testReplayEditsWrittenIntoWAL/a:1/1731767535521/Put/seqid=0 2024-11-16T14:32:15,776 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43279 is added to blk_1073741844_1020 (size=5170) 2024-11-16T14:32:15,776 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33005 is added to blk_1073741844_1020 (size=5170) 2024-11-16T14:32:15,776 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36193 is added to blk_1073741844_1020 (size=5170) 2024-11-16T14:32:15,777 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=108 B at sequenceid=2 (bloomFilter=true), to=hdfs://localhost:43457/hbase/data/default/testReplayEditsWrittenIntoWAL/01ff7f0ec04a08ce61c3eae0b0e1096b/.tmp/a/efdfc40587474910bdef33bdbccf2664 2024-11-16T14:32:15,820 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43457/hbase/data/default/testReplayEditsWrittenIntoWAL/01ff7f0ec04a08ce61c3eae0b0e1096b/.tmp/a/efdfc40587474910bdef33bdbccf2664 as hdfs://localhost:43457/hbase/data/default/testReplayEditsWrittenIntoWAL/01ff7f0ec04a08ce61c3eae0b0e1096b/a/efdfc40587474910bdef33bdbccf2664 2024-11-16T14:32:15,831 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43457/hbase/data/default/testReplayEditsWrittenIntoWAL/01ff7f0ec04a08ce61c3eae0b0e1096b/a/efdfc40587474910bdef33bdbccf2664, entries=2, sequenceid=2, filesize=5.0 K 2024-11-16T14:32:15,837 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for 01ff7f0ec04a08ce61c3eae0b0e1096b in 124ms, sequenceid=2, compaction requested=false; wal=null 2024-11-16T14:32:15,838 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:43457/hbase/data/default/testReplayEditsWrittenIntoWAL/01ff7f0ec04a08ce61c3eae0b0e1096b/recovered.edits/0000000000000000002 2024-11-16T14:32:15,839 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 01ff7f0ec04a08ce61c3eae0b0e1096b 2024-11-16T14:32:15,839 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 01ff7f0ec04a08ce61c3eae0b0e1096b 2024-11-16T14:32:15,843 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 01ff7f0ec04a08ce61c3eae0b0e1096b 2024-11-16T14:32:15,846 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43457/hbase/data/default/testReplayEditsWrittenIntoWAL/01ff7f0ec04a08ce61c3eae0b0e1096b/recovered.edits/2.seqid, newMaxSeqId=2, maxSeqId=1 2024-11-16T14:32:15,848 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 01ff7f0ec04a08ce61c3eae0b0e1096b; next sequenceid=3; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67253147, jitterRate=0.0021499842405319214}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-16T14:32:15,849 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 01ff7f0ec04a08ce61c3eae0b0e1096b: Writing region info on filesystem at 1731767535688Initializing all the Stores at 1731767535689 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731767535689Obtaining lock to block concurrent updates at 1731767535712 (+23 ms)Preparing flush snapshotting stores in 01ff7f0ec04a08ce61c3eae0b0e1096b at 1731767535712Finished memstore snapshotting testReplayEditsWrittenIntoWAL,,1731767535449.01ff7f0ec04a08ce61c3eae0b0e1096b., syncing WAL and waiting on mvcc, flushsize=dataSize=108, getHeapSize=496, getOffHeapSize=0, getCellsCount=2 at 1731767535715 (+3 ms)Flushing stores of testReplayEditsWrittenIntoWAL,,1731767535449.01ff7f0ec04a08ce61c3eae0b0e1096b. at 1731767535716 (+1 ms)Flushing 01ff7f0ec04a08ce61c3eae0b0e1096b/a: creating writer at 1731767535717 (+1 ms)Flushing 01ff7f0ec04a08ce61c3eae0b0e1096b/a: appending metadata at 1731767535751 (+34 ms)Flushing 01ff7f0ec04a08ce61c3eae0b0e1096b/a: closing flushed file at 1731767535754 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2e7f7041: reopening flushed file at 1731767535819 (+65 ms)Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for 01ff7f0ec04a08ce61c3eae0b0e1096b in 124ms, sequenceid=2, compaction requested=false; wal=null at 1731767535837 (+18 ms)Cleaning up temporary data from old regions at 1731767535839 (+2 ms)Region opened successfully at 1731767535848 (+9 ms) 2024-11-16T14:32:15,872 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayValueCompression#testNameConflictWhenSplit0 Thread=376 (was 365) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1995475656_22 at /127.0.0.1:58990 [Receiving block BP-865919460-172.17.0.2-1731767527819:blk_1073741843_1019] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-865919460-172.17.0.2-1731767527819:blk_1073741843_1019, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1995475656_22 at /127.0.0.1:42372 [Waiting for operation #10] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-8-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-0-hdfs://localhost:43457/hbase-prefix:default java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1995475656_22 at /127.0.0.1:45190 [Receiving block BP-865919460-172.17.0.2-1731767527819:blk_1073741843_1019] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: TestAsyncWALReplay-pool-0 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-865919460-172.17.0.2-1731767527819:blk_1073741843_1019, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1995475656_22 at /127.0.0.1:58926 [Waiting for operation #8] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-8-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1995475656_22 at /127.0.0.1:45076 [Waiting for operation #4] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-8-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-865919460-172.17.0.2-1731767527819:blk_1073741843_1019, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1995475656_22 at /127.0.0.1:42496 [Receiving block BP-865919460-172.17.0.2-1731767527819:blk_1073741843_1019] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=703 (was 611) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=170 (was 170), ProcessCount=11 (was 11), AvailableMemoryMB=4885 (was 4899) 2024-11-16T14:32:15,882 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayValueCompression#testNameConflictWhenSplit1 Thread=376, OpenFileDescriptor=703, MaxFileDescriptor=1048576, SystemLoadAverage=170, ProcessCount=11, AvailableMemoryMB=4882 2024-11-16T14:32:15,896 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-16T14:32:15,898 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-16T14:32:15,899 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-16T14:32:15,903 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-07890961, suffix=, logDir=hdfs://localhost:43457/hbase/WALs/hregion-07890961, archiveDir=hdfs://localhost:43457/hbase/oldWALs, maxLogs=32 2024-11-16T14:32:15,916 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-07890961/hregion-07890961.1731767535904, exclude list is [], retry=0 2024-11-16T14:32:15,920 DEBUG [AsyncFSWAL-10-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33005,DS-6a1dd528-c0be-4580-a53e-7113879f7f97,DISK] 2024-11-16T14:32:15,920 DEBUG [AsyncFSWAL-10-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36193,DS-a30e140c-8532-4096-8cae-c8d713a3d4e3,DISK] 2024-11-16T14:32:15,920 DEBUG [AsyncFSWAL-10-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43279,DS-17ab9554-4509-49b3-918b-178692b9a473,DISK] 2024-11-16T14:32:15,924 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-07890961/hregion-07890961.1731767535904 2024-11-16T14:32:15,924 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41179:41179),(127.0.0.1/127.0.0.1:35677:35677),(127.0.0.1/127.0.0.1:36015:36015)] 2024-11-16T14:32:15,925 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 07bee2f07a2ea9d68ad183bc9bd74fda, NAME => 'testReplayEditsWrittenIntoWAL,,1731767535897.07bee2f07a2ea9d68ad183bc9bd74fda.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenIntoWAL', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43457/hbase 2024-11-16T14:32:15,936 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43279 is added to blk_1073741846_1022 (size=64) 2024-11-16T14:32:15,936 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33005 is added to blk_1073741846_1022 (size=64) 2024-11-16T14:32:15,937 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36193 is added to blk_1073741846_1022 (size=64) 2024-11-16T14:32:15,938 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1731767535897.07bee2f07a2ea9d68ad183bc9bd74fda.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T14:32:15,940 INFO [StoreOpener-07bee2f07a2ea9d68ad183bc9bd74fda-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 07bee2f07a2ea9d68ad183bc9bd74fda 2024-11-16T14:32:15,942 INFO [StoreOpener-07bee2f07a2ea9d68ad183bc9bd74fda-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 07bee2f07a2ea9d68ad183bc9bd74fda columnFamilyName a 2024-11-16T14:32:15,942 DEBUG [StoreOpener-07bee2f07a2ea9d68ad183bc9bd74fda-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:32:15,943 INFO [StoreOpener-07bee2f07a2ea9d68ad183bc9bd74fda-1 {}] regionserver.HStore(327): Store=07bee2f07a2ea9d68ad183bc9bd74fda/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T14:32:15,944 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 07bee2f07a2ea9d68ad183bc9bd74fda 2024-11-16T14:32:15,945 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43457/hbase/data/default/testReplayEditsWrittenIntoWAL/07bee2f07a2ea9d68ad183bc9bd74fda 2024-11-16T14:32:15,945 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43457/hbase/data/default/testReplayEditsWrittenIntoWAL/07bee2f07a2ea9d68ad183bc9bd74fda 2024-11-16T14:32:15,946 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 07bee2f07a2ea9d68ad183bc9bd74fda 2024-11-16T14:32:15,946 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 07bee2f07a2ea9d68ad183bc9bd74fda 2024-11-16T14:32:15,948 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 07bee2f07a2ea9d68ad183bc9bd74fda 2024-11-16T14:32:15,951 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43457/hbase/data/default/testReplayEditsWrittenIntoWAL/07bee2f07a2ea9d68ad183bc9bd74fda/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-16T14:32:15,952 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 07bee2f07a2ea9d68ad183bc9bd74fda; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61257800, jitterRate=-0.08718764781951904}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-16T14:32:15,952 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 07bee2f07a2ea9d68ad183bc9bd74fda: Writing region info on filesystem at 1731767535938Initializing all the Stores at 1731767535939 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731767535939Cleaning up temporary data from old regions at 1731767535946 (+7 ms)Region opened successfully at 1731767535952 (+6 ms) 2024-11-16T14:32:15,952 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 07bee2f07a2ea9d68ad183bc9bd74fda, disabling compactions & flushes 2024-11-16T14:32:15,952 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1731767535897.07bee2f07a2ea9d68ad183bc9bd74fda. 2024-11-16T14:32:15,953 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1731767535897.07bee2f07a2ea9d68ad183bc9bd74fda. 2024-11-16T14:32:15,953 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1731767535897.07bee2f07a2ea9d68ad183bc9bd74fda. after waiting 0 ms 2024-11-16T14:32:15,953 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1731767535897.07bee2f07a2ea9d68ad183bc9bd74fda. 2024-11-16T14:32:15,953 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1731767535897.07bee2f07a2ea9d68ad183bc9bd74fda. 2024-11-16T14:32:15,953 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 07bee2f07a2ea9d68ad183bc9bd74fda: Waiting for close lock at 1731767535952Disabling compacts and flushes for region at 1731767535952Disabling writes for close at 1731767535953 (+1 ms)Writing region close event to WAL at 1731767535953Closed at 1731767535953 2024-11-16T14:32:15,960 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33005 is added to blk_1073741845_1021 (size=95) 2024-11-16T14:32:15,960 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36193 is added to blk_1073741845_1021 (size=95) 2024-11-16T14:32:15,960 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43279 is added to blk_1073741845_1021 (size=95) 2024-11-16T14:32:15,963 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-16T14:32:15,963 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-07890961:(num 1731767535904) 2024-11-16T14:32:15,964 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-11-16T14:32:15,975 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36193 is added to blk_1073741847_1023 (size=320) 2024-11-16T14:32:15,975 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33005 is added to blk_1073741847_1023 (size=320) 2024-11-16T14:32:15,976 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43279 is added to blk_1073741847_1023 (size=320) 2024-11-16T14:32:15,977 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-11-16T14:32:15,987 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36193 is added to blk_1073741848_1024 (size=253) 2024-11-16T14:32:15,987 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33005 is added to blk_1073741848_1024 (size=253) 2024-11-16T14:32:15,988 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43279 is added to blk_1073741848_1024 (size=253) 2024-11-16T14:32:16,003 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:43457/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731767535895/wal-2, size=253 (253bytes) 2024-11-16T14:32:16,004 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:43457/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731767535895/wal-2 2024-11-16T14:32:16,005 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:43457/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731767535895/wal-2 after 0ms 2024-11-16T14:32:16,009 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:43457/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731767535895/wal-2: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-16T14:32:16,009 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:43457/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731767535895/wal-2 took 6ms 2024-11-16T14:32:16,013 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:43457/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731767535895/wal-2 so closing down 2024-11-16T14:32:16,013 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-16T14:32:16,016 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000002-wal-2.temp 2024-11-16T14:32:16,018 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:43457/hbase/data/default/testReplayEditsWrittenIntoWAL/07bee2f07a2ea9d68ad183bc9bd74fda/recovered.edits/0000000000000000002-wal-2.temp 2024-11-16T14:32:16,018 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-16T14:32:16,027 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36193 is added to blk_1073741849_1025 (size=253) 2024-11-16T14:32:16,027 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33005 is added to blk_1073741849_1025 (size=253) 2024-11-16T14:32:16,027 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43279 is added to blk_1073741849_1025 (size=253) 2024-11-16T14:32:16,028 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:43457/hbase/data/default/testReplayEditsWrittenIntoWAL/07bee2f07a2ea9d68ad183bc9bd74fda/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-11-16T14:32:16,030 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:43457/hbase/data/default/testReplayEditsWrittenIntoWAL/07bee2f07a2ea9d68ad183bc9bd74fda/recovered.edits/0000000000000000002-wal-2.temp to hdfs://localhost:43457/hbase/data/default/testReplayEditsWrittenIntoWAL/07bee2f07a2ea9d68ad183bc9bd74fda/recovered.edits/0000000000000000002 2024-11-16T14:32:16,030 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 1 edits across 1 Regions in 19 ms; skipped=0; WAL=hdfs://localhost:43457/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731767535895/wal-2, size=253, length=253, corrupted=false, cancelled=false 2024-11-16T14:32:16,030 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:43457/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731767535895/wal-2, journal: Splitting hdfs://localhost:43457/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731767535895/wal-2, size=253 (253bytes) at 1731767536004Finishing writing output for hdfs://localhost:43457/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731767535895/wal-2 so closing down at 1731767536013 (+9 ms)Creating recovered edits writer path=hdfs://localhost:43457/hbase/data/default/testReplayEditsWrittenIntoWAL/07bee2f07a2ea9d68ad183bc9bd74fda/recovered.edits/0000000000000000002-wal-2.temp at 1731767536018 (+5 ms)3 split writer threads finished at 1731767536018Closed recovered edits writer path=hdfs://localhost:43457/hbase/data/default/testReplayEditsWrittenIntoWAL/07bee2f07a2ea9d68ad183bc9bd74fda/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1731767536028 (+10 ms)Rename recovered edits hdfs://localhost:43457/hbase/data/default/testReplayEditsWrittenIntoWAL/07bee2f07a2ea9d68ad183bc9bd74fda/recovered.edits/0000000000000000002-wal-2.temp to hdfs://localhost:43457/hbase/data/default/testReplayEditsWrittenIntoWAL/07bee2f07a2ea9d68ad183bc9bd74fda/recovered.edits/0000000000000000002 at 1731767536030 (+2 ms)Processed 1 edits across 1 Regions in 19 ms; skipped=0; WAL=hdfs://localhost:43457/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731767535895/wal-2, size=253, length=253, corrupted=false, cancelled=false at 1731767536030 2024-11-16T14:32:16,044 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:43457/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731767535895/wal-1, size=320 (320bytes) 2024-11-16T14:32:16,044 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:43457/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731767535895/wal-1 2024-11-16T14:32:16,044 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:43457/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731767535895/wal-1 after 0ms 2024-11-16T14:32:16,048 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:43457/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731767535895/wal-1: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-16T14:32:16,048 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:43457/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731767535895/wal-1 took 5ms 2024-11-16T14:32:16,052 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:43457/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731767535895/wal-1 so closing down 2024-11-16T14:32:16,052 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-16T14:32:16,054 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal-1.temp 2024-11-16T14:32:16,056 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:43457/hbase/data/default/testReplayEditsWrittenIntoWAL/07bee2f07a2ea9d68ad183bc9bd74fda/recovered.edits/0000000000000000001-wal-1.temp 2024-11-16T14:32:16,056 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-16T14:32:16,063 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33005 is added to blk_1073741850_1026 (size=320) 2024-11-16T14:32:16,063 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43279 is added to blk_1073741850_1026 (size=320) 2024-11-16T14:32:16,064 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36193 is added to blk_1073741850_1026 (size=320) 2024-11-16T14:32:16,064 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:43457/hbase/data/default/testReplayEditsWrittenIntoWAL/07bee2f07a2ea9d68ad183bc9bd74fda/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) 2024-11-16T14:32:16,068 DEBUG [split-log-closeStream-pool-0 {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:43457/hbase/data/default/testReplayEditsWrittenIntoWAL/07bee2f07a2ea9d68ad183bc9bd74fda/recovered.edits/0000000000000000002: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-16T14:32:16,070 WARN [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(175): Found existing old edits file. It could be the result of a previous failed split attempt or we have duplicated wal entries. Deleting hdfs://localhost:43457/hbase/data/default/testReplayEditsWrittenIntoWAL/07bee2f07a2ea9d68ad183bc9bd74fda/recovered.edits/0000000000000000002, length=253 2024-11-16T14:32:16,073 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:43457/hbase/data/default/testReplayEditsWrittenIntoWAL/07bee2f07a2ea9d68ad183bc9bd74fda/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:43457/hbase/data/default/testReplayEditsWrittenIntoWAL/07bee2f07a2ea9d68ad183bc9bd74fda/recovered.edits/0000000000000000002 2024-11-16T14:32:16,073 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 2 edits across 1 Regions in 24 ms; skipped=0; WAL=hdfs://localhost:43457/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731767535895/wal-1, size=320, length=320, corrupted=false, cancelled=false 2024-11-16T14:32:16,073 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:43457/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731767535895/wal-1, journal: Splitting hdfs://localhost:43457/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731767535895/wal-1, size=320 (320bytes) at 1731767536044Finishing writing output for hdfs://localhost:43457/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731767535895/wal-1 so closing down at 1731767536052 (+8 ms)Creating recovered edits writer path=hdfs://localhost:43457/hbase/data/default/testReplayEditsWrittenIntoWAL/07bee2f07a2ea9d68ad183bc9bd74fda/recovered.edits/0000000000000000001-wal-1.temp at 1731767536056 (+4 ms)3 split writer threads finished at 1731767536056Closed recovered edits writer path=hdfs://localhost:43457/hbase/data/default/testReplayEditsWrittenIntoWAL/07bee2f07a2ea9d68ad183bc9bd74fda/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) at 1731767536064 (+8 ms)Rename recovered edits hdfs://localhost:43457/hbase/data/default/testReplayEditsWrittenIntoWAL/07bee2f07a2ea9d68ad183bc9bd74fda/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:43457/hbase/data/default/testReplayEditsWrittenIntoWAL/07bee2f07a2ea9d68ad183bc9bd74fda/recovered.edits/0000000000000000002 at 1731767536073 (+9 ms)Processed 2 edits across 1 Regions in 24 ms; skipped=0; WAL=hdfs://localhost:43457/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731767535895/wal-1, size=320, length=320, corrupted=false, cancelled=false at 1731767536073 2024-11-16T14:32:16,073 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-16T14:32:16,075 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:43457/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731767535895, archiveDir=hdfs://localhost:43457/hbase/oldWALs, maxLogs=32 2024-11-16T14:32:16,090 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testnameconflictwhensplit1-manual,16010,1731767535895/wal.1731767536076, exclude list is [], retry=0 2024-11-16T14:32:16,093 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33005,DS-6a1dd528-c0be-4580-a53e-7113879f7f97,DISK] 2024-11-16T14:32:16,094 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36193,DS-a30e140c-8532-4096-8cae-c8d713a3d4e3,DISK] 2024-11-16T14:32:16,094 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43279,DS-17ab9554-4509-49b3-918b-178692b9a473,DISK] 2024-11-16T14:32:16,096 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testnameconflictwhensplit1-manual,16010,1731767535895/wal.1731767536076 2024-11-16T14:32:16,096 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41179:41179),(127.0.0.1/127.0.0.1:35677:35677),(127.0.0.1/127.0.0.1:36015:36015)] 2024-11-16T14:32:16,096 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 07bee2f07a2ea9d68ad183bc9bd74fda, NAME => 'testReplayEditsWrittenIntoWAL,,1731767535897.07bee2f07a2ea9d68ad183bc9bd74fda.', STARTKEY => '', ENDKEY => ''} 2024-11-16T14:32:16,097 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1731767535897.07bee2f07a2ea9d68ad183bc9bd74fda.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T14:32:16,097 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 07bee2f07a2ea9d68ad183bc9bd74fda 2024-11-16T14:32:16,097 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 07bee2f07a2ea9d68ad183bc9bd74fda 2024-11-16T14:32:16,099 INFO [StoreOpener-07bee2f07a2ea9d68ad183bc9bd74fda-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 07bee2f07a2ea9d68ad183bc9bd74fda 2024-11-16T14:32:16,101 INFO [StoreOpener-07bee2f07a2ea9d68ad183bc9bd74fda-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 07bee2f07a2ea9d68ad183bc9bd74fda columnFamilyName a 2024-11-16T14:32:16,101 DEBUG [StoreOpener-07bee2f07a2ea9d68ad183bc9bd74fda-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:32:16,102 INFO [StoreOpener-07bee2f07a2ea9d68ad183bc9bd74fda-1 {}] regionserver.HStore(327): Store=07bee2f07a2ea9d68ad183bc9bd74fda/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T14:32:16,102 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 07bee2f07a2ea9d68ad183bc9bd74fda 2024-11-16T14:32:16,103 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43457/hbase/data/default/testReplayEditsWrittenIntoWAL/07bee2f07a2ea9d68ad183bc9bd74fda 2024-11-16T14:32:16,105 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:43457/hbase/data/default/testReplayEditsWrittenIntoWAL/07bee2f07a2ea9d68ad183bc9bd74fda 2024-11-16T14:32:16,106 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:43457/hbase/data/default/testReplayEditsWrittenIntoWAL/07bee2f07a2ea9d68ad183bc9bd74fda/recovered.edits/0000000000000000002 2024-11-16T14:32:16,110 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:43457/hbase/data/default/testReplayEditsWrittenIntoWAL/07bee2f07a2ea9d68ad183bc9bd74fda/recovered.edits/0000000000000000002: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-16T14:32:16,112 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 2, skipped 0, firstSequenceIdInLog=1, maxSequenceIdInLog=2, path=hdfs://localhost:43457/hbase/data/default/testReplayEditsWrittenIntoWAL/07bee2f07a2ea9d68ad183bc9bd74fda/recovered.edits/0000000000000000002 2024-11-16T14:32:16,113 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 07bee2f07a2ea9d68ad183bc9bd74fda 1/1 column families, dataSize=108 B heapSize=512 B 2024-11-16T14:32:16,132 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43457/hbase/data/default/testReplayEditsWrittenIntoWAL/07bee2f07a2ea9d68ad183bc9bd74fda/.tmp/a/558ad9b6ca594121a0e38b67aa5aa373 is 58, key is testReplayEditsWrittenIntoWAL/a:1/1731767535963/Put/seqid=0 2024-11-16T14:32:16,140 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33005 is added to blk_1073741852_1028 (size=5170) 2024-11-16T14:32:16,141 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43279 is added to blk_1073741852_1028 (size=5170) 2024-11-16T14:32:16,141 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36193 is added to blk_1073741852_1028 (size=5170) 2024-11-16T14:32:16,141 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=108 B at sequenceid=2 (bloomFilter=true), to=hdfs://localhost:43457/hbase/data/default/testReplayEditsWrittenIntoWAL/07bee2f07a2ea9d68ad183bc9bd74fda/.tmp/a/558ad9b6ca594121a0e38b67aa5aa373 2024-11-16T14:32:16,151 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43457/hbase/data/default/testReplayEditsWrittenIntoWAL/07bee2f07a2ea9d68ad183bc9bd74fda/.tmp/a/558ad9b6ca594121a0e38b67aa5aa373 as hdfs://localhost:43457/hbase/data/default/testReplayEditsWrittenIntoWAL/07bee2f07a2ea9d68ad183bc9bd74fda/a/558ad9b6ca594121a0e38b67aa5aa373 2024-11-16T14:32:16,162 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43457/hbase/data/default/testReplayEditsWrittenIntoWAL/07bee2f07a2ea9d68ad183bc9bd74fda/a/558ad9b6ca594121a0e38b67aa5aa373, entries=2, sequenceid=2, filesize=5.0 K 2024-11-16T14:32:16,163 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for 07bee2f07a2ea9d68ad183bc9bd74fda in 50ms, sequenceid=2, compaction requested=false; wal=null 2024-11-16T14:32:16,164 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:43457/hbase/data/default/testReplayEditsWrittenIntoWAL/07bee2f07a2ea9d68ad183bc9bd74fda/recovered.edits/0000000000000000002 2024-11-16T14:32:16,164 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 07bee2f07a2ea9d68ad183bc9bd74fda 2024-11-16T14:32:16,165 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 07bee2f07a2ea9d68ad183bc9bd74fda 2024-11-16T14:32:16,168 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 07bee2f07a2ea9d68ad183bc9bd74fda 2024-11-16T14:32:16,171 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43457/hbase/data/default/testReplayEditsWrittenIntoWAL/07bee2f07a2ea9d68ad183bc9bd74fda/recovered.edits/2.seqid, newMaxSeqId=2, maxSeqId=1 2024-11-16T14:32:16,173 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 07bee2f07a2ea9d68ad183bc9bd74fda; next sequenceid=3; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70145450, jitterRate=0.04524865746498108}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-16T14:32:16,173 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 07bee2f07a2ea9d68ad183bc9bd74fda: Writing region info on filesystem at 1731767536097Initializing all the Stores at 1731767536098 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731767536098Obtaining lock to block concurrent updates at 1731767536113 (+15 ms)Preparing flush snapshotting stores in 07bee2f07a2ea9d68ad183bc9bd74fda at 1731767536113Finished memstore snapshotting testReplayEditsWrittenIntoWAL,,1731767535897.07bee2f07a2ea9d68ad183bc9bd74fda., syncing WAL and waiting on mvcc, flushsize=dataSize=108, getHeapSize=496, getOffHeapSize=0, getCellsCount=2 at 1731767536113Flushing stores of testReplayEditsWrittenIntoWAL,,1731767535897.07bee2f07a2ea9d68ad183bc9bd74fda. at 1731767536113Flushing 07bee2f07a2ea9d68ad183bc9bd74fda/a: creating writer at 1731767536113Flushing 07bee2f07a2ea9d68ad183bc9bd74fda/a: appending metadata at 1731767536130 (+17 ms)Flushing 07bee2f07a2ea9d68ad183bc9bd74fda/a: closing flushed file at 1731767536131 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3c20606c: reopening flushed file at 1731767536150 (+19 ms)Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for 07bee2f07a2ea9d68ad183bc9bd74fda in 50ms, sequenceid=2, compaction requested=false; wal=null at 1731767536163 (+13 ms)Cleaning up temporary data from old regions at 1731767536165 (+2 ms)Region opened successfully at 1731767536173 (+8 ms) 2024-11-16T14:32:16,189 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayValueCompression#testNameConflictWhenSplit1 Thread=386 (was 376) Potentially hanging thread: PacketResponder: BP-865919460-172.17.0.2-1731767527819:blk_1073741851_1027, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-865919460-172.17.0.2-1731767527819:blk_1073741851_1027, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-865919460-172.17.0.2-1731767527819:blk_1073741851_1027, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-10-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-10-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1995475656_22 at /127.0.0.1:45076 [Waiting for operation #8] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-10-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1995475656_22 at /127.0.0.1:42574 [Receiving block BP-865919460-172.17.0.2-1731767527819:blk_1073741851_1027] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1995475656_22 at /127.0.0.1:42372 [Waiting for operation #19] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1995475656_22 at /127.0.0.1:59046 [Receiving block BP-865919460-172.17.0.2-1731767527819:blk_1073741851_1027] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1995475656_22 at /127.0.0.1:45270 [Receiving block BP-865919460-172.17.0.2-1731767527819:blk_1073741851_1027] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1995475656_22 at /127.0.0.1:58926 [Waiting for operation #10] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=785 (was 703) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=170 (was 170), ProcessCount=11 (was 11), AvailableMemoryMB=4868 (was 4882) 2024-11-16T14:32:16,200 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayValueCompression#testReplayEditsWrittenIntoWAL Thread=386, OpenFileDescriptor=785, MaxFileDescriptor=1048576, SystemLoadAverage=170, ProcessCount=11, AvailableMemoryMB=4867 2024-11-16T14:32:16,216 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-16T14:32:16,218 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-16T14:32:16,219 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-16T14:32:16,222 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-81866887, suffix=, logDir=hdfs://localhost:43457/hbase/WALs/hregion-81866887, archiveDir=hdfs://localhost:43457/hbase/oldWALs, maxLogs=32 2024-11-16T14:32:16,235 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-81866887/hregion-81866887.1731767536223, exclude list is [], retry=0 2024-11-16T14:32:16,239 DEBUG [AsyncFSWAL-12-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36193,DS-a30e140c-8532-4096-8cae-c8d713a3d4e3,DISK] 2024-11-16T14:32:16,239 DEBUG [AsyncFSWAL-12-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33005,DS-6a1dd528-c0be-4580-a53e-7113879f7f97,DISK] 2024-11-16T14:32:16,239 DEBUG [AsyncFSWAL-12-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43279,DS-17ab9554-4509-49b3-918b-178692b9a473,DISK] 2024-11-16T14:32:16,241 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-81866887/hregion-81866887.1731767536223 2024-11-16T14:32:16,242 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35677:35677),(127.0.0.1/127.0.0.1:41179:41179),(127.0.0.1/127.0.0.1:36015:36015)] 2024-11-16T14:32:16,242 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 67a315f8951249f5de2391507bf27cca, NAME => 'testReplayEditsWrittenIntoWAL,,1731767536217.67a315f8951249f5de2391507bf27cca.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenIntoWAL', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43457/hbase 2024-11-16T14:32:16,251 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33005 is added to blk_1073741854_1030 (size=64) 2024-11-16T14:32:16,252 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36193 is added to blk_1073741854_1030 (size=64) 2024-11-16T14:32:16,252 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43279 is added to blk_1073741854_1030 (size=64) 2024-11-16T14:32:16,253 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1731767536217.67a315f8951249f5de2391507bf27cca.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T14:32:16,256 INFO [StoreOpener-67a315f8951249f5de2391507bf27cca-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 67a315f8951249f5de2391507bf27cca 2024-11-16T14:32:16,258 INFO [StoreOpener-67a315f8951249f5de2391507bf27cca-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 67a315f8951249f5de2391507bf27cca columnFamilyName a 2024-11-16T14:32:16,258 DEBUG [StoreOpener-67a315f8951249f5de2391507bf27cca-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:32:16,258 INFO [StoreOpener-67a315f8951249f5de2391507bf27cca-1 {}] regionserver.HStore(327): Store=67a315f8951249f5de2391507bf27cca/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T14:32:16,258 INFO [StoreOpener-67a315f8951249f5de2391507bf27cca-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 67a315f8951249f5de2391507bf27cca 2024-11-16T14:32:16,260 INFO [StoreOpener-67a315f8951249f5de2391507bf27cca-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 67a315f8951249f5de2391507bf27cca columnFamilyName b 2024-11-16T14:32:16,260 DEBUG [StoreOpener-67a315f8951249f5de2391507bf27cca-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:32:16,261 INFO [StoreOpener-67a315f8951249f5de2391507bf27cca-1 {}] regionserver.HStore(327): Store=67a315f8951249f5de2391507bf27cca/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T14:32:16,261 INFO [StoreOpener-67a315f8951249f5de2391507bf27cca-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 67a315f8951249f5de2391507bf27cca 2024-11-16T14:32:16,263 INFO [StoreOpener-67a315f8951249f5de2391507bf27cca-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 67a315f8951249f5de2391507bf27cca columnFamilyName c 2024-11-16T14:32:16,263 DEBUG [StoreOpener-67a315f8951249f5de2391507bf27cca-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:32:16,264 INFO [StoreOpener-67a315f8951249f5de2391507bf27cca-1 {}] regionserver.HStore(327): Store=67a315f8951249f5de2391507bf27cca/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T14:32:16,264 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 67a315f8951249f5de2391507bf27cca 2024-11-16T14:32:16,265 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43457/hbase/data/default/testReplayEditsWrittenIntoWAL/67a315f8951249f5de2391507bf27cca 2024-11-16T14:32:16,265 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43457/hbase/data/default/testReplayEditsWrittenIntoWAL/67a315f8951249f5de2391507bf27cca 2024-11-16T14:32:16,267 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 67a315f8951249f5de2391507bf27cca 2024-11-16T14:32:16,267 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 67a315f8951249f5de2391507bf27cca 2024-11-16T14:32:16,267 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenIntoWAL descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-16T14:32:16,268 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 67a315f8951249f5de2391507bf27cca 2024-11-16T14:32:16,271 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43457/hbase/data/default/testReplayEditsWrittenIntoWAL/67a315f8951249f5de2391507bf27cca/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-16T14:32:16,271 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 67a315f8951249f5de2391507bf27cca; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73067562, jitterRate=0.08879151940345764}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-16T14:32:16,272 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 67a315f8951249f5de2391507bf27cca: Writing region info on filesystem at 1731767536253Initializing all the Stores at 1731767536255 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731767536255Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731767536256 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731767536256Cleaning up temporary data from old regions at 1731767536267 (+11 ms)Region opened successfully at 1731767536271 (+4 ms) 2024-11-16T14:32:16,272 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 67a315f8951249f5de2391507bf27cca, disabling compactions & flushes 2024-11-16T14:32:16,272 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1731767536217.67a315f8951249f5de2391507bf27cca. 2024-11-16T14:32:16,272 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1731767536217.67a315f8951249f5de2391507bf27cca. 2024-11-16T14:32:16,272 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1731767536217.67a315f8951249f5de2391507bf27cca. after waiting 0 ms 2024-11-16T14:32:16,272 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1731767536217.67a315f8951249f5de2391507bf27cca. 2024-11-16T14:32:16,272 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1731767536217.67a315f8951249f5de2391507bf27cca. 2024-11-16T14:32:16,272 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 67a315f8951249f5de2391507bf27cca: Waiting for close lock at 1731767536272Disabling compacts and flushes for region at 1731767536272Disabling writes for close at 1731767536272Writing region close event to WAL at 1731767536272Closed at 1731767536272 2024-11-16T14:32:16,275 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36193 is added to blk_1073741853_1029 (size=95) 2024-11-16T14:32:16,276 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33005 is added to blk_1073741853_1029 (size=95) 2024-11-16T14:32:16,276 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43279 is added to blk_1073741853_1029 (size=95) 2024-11-16T14:32:16,280 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-16T14:32:16,280 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-81866887:(num 1731767536223) 2024-11-16T14:32:16,280 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-16T14:32:16,283 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:43457/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731767536215, archiveDir=hdfs://localhost:43457/hbase/oldWALs, maxLogs=32 2024-11-16T14:32:16,297 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731767536215/wal.1731767536284, exclude list is [], retry=0 2024-11-16T14:32:16,301 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36193,DS-a30e140c-8532-4096-8cae-c8d713a3d4e3,DISK] 2024-11-16T14:32:16,301 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33005,DS-6a1dd528-c0be-4580-a53e-7113879f7f97,DISK] 2024-11-16T14:32:16,302 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43279,DS-17ab9554-4509-49b3-918b-178692b9a473,DISK] 2024-11-16T14:32:16,303 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731767536215/wal.1731767536284 2024-11-16T14:32:16,304 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35677:35677),(127.0.0.1/127.0.0.1:41179:41179),(127.0.0.1/127.0.0.1:36015:36015)] 2024-11-16T14:32:16,569 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:43457/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731767536215/wal.1731767536284, size=0 (0bytes) 2024-11-16T14:32:16,569 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:43457/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731767536215/wal.1731767536284 might be still open, length is 0 2024-11-16T14:32:16,569 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:43457/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731767536215/wal.1731767536284 2024-11-16T14:32:16,571 WARN [IPC Server handler 0 on default port 43457 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731767536215/wal.1731767536284 has not been closed. Lease recovery is in progress. RecoveryId = 1032 for block blk_1073741855_1031 2024-11-16T14:32:16,571 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:43457/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731767536215/wal.1731767536284 after 2ms 2024-11-16T14:32:17,266 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1995475656_22 at /127.0.0.1:45292 [Receiving block BP-865919460-172.17.0.2-1731767527819:blk_1073741855_1031] {}] datanode.DataXceiver(331): 127.0.0.1:36193:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:45292 dst: /127.0.0.1:36193 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:36193 remote=/127.0.0.1:45292]. Total timeout mills is 60000, 59271 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:32:17,267 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1995475656_22 at /127.0.0.1:59080 [Receiving block BP-865919460-172.17.0.2-1731767527819:blk_1073741855_1031] {}] datanode.DataXceiver(331): 127.0.0.1:43279:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:59080 dst: /127.0.0.1:43279 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:32:17,267 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1995475656_22 at /127.0.0.1:42612 [Receiving block BP-865919460-172.17.0.2-1731767527819:blk_1073741855_1031] {}] datanode.DataXceiver(331): 127.0.0.1:33005:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:42612 dst: /127.0.0.1:33005 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:32:17,280 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36193 is added to blk_1073741855_1032 (size=263633) 2024-11-16T14:32:17,282 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33005 is added to blk_1073741855_1032 (size=263633) 2024-11-16T14:32:17,284 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43279 is added to blk_1073741855_1032 (size=263633) 2024-11-16T14:32:20,557 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-16T14:32:20,572 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:43457/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731767536215/wal.1731767536284 after 4003ms 2024-11-16T14:32:20,576 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:43457/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731767536215/wal.1731767536284: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-16T14:32:20,578 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:43457/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731767536215/wal.1731767536284 took 4009ms 2024-11-16T14:32:20,583 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal.1731767536284.temp 2024-11-16T14:32:20,588 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:43457/hbase/data/default/testReplayEditsWrittenIntoWAL/67a315f8951249f5de2391507bf27cca/recovered.edits/0000000000000000001-wal.1731767536284.temp 2024-11-16T14:32:20,629 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-16T14:32:20,735 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:43457/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731767536215/wal.1731767536284; continuing. 2024-11-16T14:32:20,735 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:43457/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731767536215/wal.1731767536284 so closing down 2024-11-16T14:32:20,735 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-16T14:32:20,735 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-16T14:32:20,738 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43279 is added to blk_1073741856_1033 (size=263641) 2024-11-16T14:32:20,739 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33005 is added to blk_1073741856_1033 (size=263641) 2024-11-16T14:32:20,739 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36193 is added to blk_1073741856_1033 (size=263641) 2024-11-16T14:32:20,740 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:43457/hbase/data/default/testReplayEditsWrittenIntoWAL/67a315f8951249f5de2391507bf27cca/recovered.edits/0000000000000000001-wal.1731767536284.temp (wrote 3002 edits, skipped 0 edits in 69 ms) 2024-11-16T14:32:20,741 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:43457/hbase/data/default/testReplayEditsWrittenIntoWAL/67a315f8951249f5de2391507bf27cca/recovered.edits/0000000000000000001-wal.1731767536284.temp to hdfs://localhost:43457/hbase/data/default/testReplayEditsWrittenIntoWAL/67a315f8951249f5de2391507bf27cca/recovered.edits/0000000000000003002 2024-11-16T14:32:20,742 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 3002 edits across 1 Regions in 163 ms; skipped=0; WAL=hdfs://localhost:43457/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731767536215/wal.1731767536284, size=0, length=0, corrupted=false, cancelled=false 2024-11-16T14:32:20,742 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:43457/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731767536215/wal.1731767536284, journal: Splitting hdfs://localhost:43457/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731767536215/wal.1731767536284, size=0 (0bytes) at 1731767536569Creating recovered edits writer path=hdfs://localhost:43457/hbase/data/default/testReplayEditsWrittenIntoWAL/67a315f8951249f5de2391507bf27cca/recovered.edits/0000000000000000001-wal.1731767536284.temp at 1731767540588 (+4019 ms)Split 1024 edits, skipped 0 edits. at 1731767540671 (+83 ms)Split 2048 edits, skipped 0 edits. at 1731767540706 (+35 ms)Finishing writing output for hdfs://localhost:43457/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731767536215/wal.1731767536284 so closing down at 1731767540735 (+29 ms)3 split writer threads finished at 1731767540735Closed recovered edits writer path=hdfs://localhost:43457/hbase/data/default/testReplayEditsWrittenIntoWAL/67a315f8951249f5de2391507bf27cca/recovered.edits/0000000000000000001-wal.1731767536284.temp (wrote 3002 edits, skipped 0 edits in 69 ms) at 1731767540740 (+5 ms)Rename recovered edits hdfs://localhost:43457/hbase/data/default/testReplayEditsWrittenIntoWAL/67a315f8951249f5de2391507bf27cca/recovered.edits/0000000000000000001-wal.1731767536284.temp to hdfs://localhost:43457/hbase/data/default/testReplayEditsWrittenIntoWAL/67a315f8951249f5de2391507bf27cca/recovered.edits/0000000000000003002 at 1731767540742 (+2 ms)Processed 3002 edits across 1 Regions in 163 ms; skipped=0; WAL=hdfs://localhost:43457/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731767536215/wal.1731767536284, size=0, length=0, corrupted=false, cancelled=false at 1731767540742 2024-11-16T14:32:20,744 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:43457/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731767536215/wal.1731767536284 to hdfs://localhost:43457/hbase/oldWALs/wal.1731767536284 2024-11-16T14:32:20,746 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:43457/hbase/data/default/testReplayEditsWrittenIntoWAL/67a315f8951249f5de2391507bf27cca/recovered.edits/0000000000000003002 2024-11-16T14:32:20,746 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-16T14:32:20,748 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:43457/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731767536215, archiveDir=hdfs://localhost:43457/hbase/oldWALs, maxLogs=32 2024-11-16T14:32:20,762 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731767536215/wal.1731767540749, exclude list is [], retry=0 2024-11-16T14:32:20,765 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36193,DS-a30e140c-8532-4096-8cae-c8d713a3d4e3,DISK] 2024-11-16T14:32:20,765 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43279,DS-17ab9554-4509-49b3-918b-178692b9a473,DISK] 2024-11-16T14:32:20,766 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33005,DS-6a1dd528-c0be-4580-a53e-7113879f7f97,DISK] 2024-11-16T14:32:20,768 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731767536215/wal.1731767540749 2024-11-16T14:32:20,768 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35677:35677),(127.0.0.1/127.0.0.1:36015:36015),(127.0.0.1/127.0.0.1:41179:41179)] 2024-11-16T14:32:20,769 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1731767536217.67a315f8951249f5de2391507bf27cca.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T14:32:20,771 INFO [StoreOpener-67a315f8951249f5de2391507bf27cca-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 67a315f8951249f5de2391507bf27cca 2024-11-16T14:32:20,772 INFO [StoreOpener-67a315f8951249f5de2391507bf27cca-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 67a315f8951249f5de2391507bf27cca columnFamilyName a 2024-11-16T14:32:20,772 DEBUG [StoreOpener-67a315f8951249f5de2391507bf27cca-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:32:20,773 INFO [StoreOpener-67a315f8951249f5de2391507bf27cca-1 {}] regionserver.HStore(327): Store=67a315f8951249f5de2391507bf27cca/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T14:32:20,773 INFO [StoreOpener-67a315f8951249f5de2391507bf27cca-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 67a315f8951249f5de2391507bf27cca 2024-11-16T14:32:20,774 INFO [StoreOpener-67a315f8951249f5de2391507bf27cca-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 67a315f8951249f5de2391507bf27cca columnFamilyName b 2024-11-16T14:32:20,775 DEBUG [StoreOpener-67a315f8951249f5de2391507bf27cca-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:32:20,775 INFO [StoreOpener-67a315f8951249f5de2391507bf27cca-1 {}] regionserver.HStore(327): Store=67a315f8951249f5de2391507bf27cca/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T14:32:20,775 INFO [StoreOpener-67a315f8951249f5de2391507bf27cca-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 67a315f8951249f5de2391507bf27cca 2024-11-16T14:32:20,777 INFO [StoreOpener-67a315f8951249f5de2391507bf27cca-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 67a315f8951249f5de2391507bf27cca columnFamilyName c 2024-11-16T14:32:20,777 DEBUG [StoreOpener-67a315f8951249f5de2391507bf27cca-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:32:20,778 INFO [StoreOpener-67a315f8951249f5de2391507bf27cca-1 {}] regionserver.HStore(327): Store=67a315f8951249f5de2391507bf27cca/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T14:32:20,778 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 67a315f8951249f5de2391507bf27cca 2024-11-16T14:32:20,779 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43457/hbase/data/default/testReplayEditsWrittenIntoWAL/67a315f8951249f5de2391507bf27cca 2024-11-16T14:32:20,781 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:43457/hbase/data/default/testReplayEditsWrittenIntoWAL/67a315f8951249f5de2391507bf27cca 2024-11-16T14:32:20,782 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:43457/hbase/data/default/testReplayEditsWrittenIntoWAL/67a315f8951249f5de2391507bf27cca/recovered.edits/0000000000000003002 2024-11-16T14:32:20,785 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:43457/hbase/data/default/testReplayEditsWrittenIntoWAL/67a315f8951249f5de2391507bf27cca/recovered.edits/0000000000000003002: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-16T14:32:20,843 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-11-16T14:32:21,178 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 67a315f8951249f5de2391507bf27cca 3/3 column families, dataSize=42.49 KB heapSize=100.11 KB 2024-11-16T14:32:21,222 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43457/hbase/data/default/testReplayEditsWrittenIntoWAL/67a315f8951249f5de2391507bf27cca/.tmp/a/078645883e3242f79e9589d11a752fc0 is 62, key is testReplayEditsWrittenIntoWAL/a:100/1731767536312/Put/seqid=0 2024-11-16T14:32:21,232 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43279 is added to blk_1073741858_1035 (size=50463) 2024-11-16T14:32:21,232 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33005 is added to blk_1073741858_1035 (size=50463) 2024-11-16T14:32:21,232 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36193 is added to blk_1073741858_1035 (size=50463) 2024-11-16T14:32:21,233 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=754 (bloomFilter=true), to=hdfs://localhost:43457/hbase/data/default/testReplayEditsWrittenIntoWAL/67a315f8951249f5de2391507bf27cca/.tmp/a/078645883e3242f79e9589d11a752fc0 2024-11-16T14:32:21,241 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43457/hbase/data/default/testReplayEditsWrittenIntoWAL/67a315f8951249f5de2391507bf27cca/.tmp/a/078645883e3242f79e9589d11a752fc0 as hdfs://localhost:43457/hbase/data/default/testReplayEditsWrittenIntoWAL/67a315f8951249f5de2391507bf27cca/a/078645883e3242f79e9589d11a752fc0 2024-11-16T14:32:21,248 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43457/hbase/data/default/testReplayEditsWrittenIntoWAL/67a315f8951249f5de2391507bf27cca/a/078645883e3242f79e9589d11a752fc0, entries=754, sequenceid=754, filesize=49.3 K 2024-11-16T14:32:21,249 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~42.49 KB/43512, heapSize ~99.59 KB/101984, currentSize=0 B/0 for 67a315f8951249f5de2391507bf27cca in 70ms, sequenceid=754, compaction requested=false; wal=null 2024-11-16T14:32:21,290 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-11-16T14:32:21,290 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 67a315f8951249f5de2391507bf27cca 3/3 column families, dataSize=42.49 KB heapSize=100.11 KB 2024-11-16T14:32:21,300 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43457/hbase/data/default/testReplayEditsWrittenIntoWAL/67a315f8951249f5de2391507bf27cca/.tmp/a/b122abb60b364858875e3c83a80a0d07 is 62, key is testReplayEditsWrittenIntoWAL/a:754/1731767536378/Put/seqid=0 2024-11-16T14:32:21,309 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33005 is added to blk_1073741859_1036 (size=20072) 2024-11-16T14:32:21,310 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36193 is added to blk_1073741859_1036 (size=20072) 2024-11-16T14:32:21,310 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43279 is added to blk_1073741859_1036 (size=20072) 2024-11-16T14:32:21,311 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.93 KB at sequenceid=1508 (bloomFilter=true), to=hdfs://localhost:43457/hbase/data/default/testReplayEditsWrittenIntoWAL/67a315f8951249f5de2391507bf27cca/.tmp/a/b122abb60b364858875e3c83a80a0d07 2024-11-16T14:32:21,344 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43457/hbase/data/default/testReplayEditsWrittenIntoWAL/67a315f8951249f5de2391507bf27cca/.tmp/b/5126ac70253a49f4aeecaff44e4da77f is 62, key is testReplayEditsWrittenIntoWAL/b:100/1731767536407/Put/seqid=0 2024-11-16T14:32:21,352 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33005 is added to blk_1073741860_1037 (size=35835) 2024-11-16T14:32:21,352 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43279 is added to blk_1073741860_1037 (size=35835) 2024-11-16T14:32:21,352 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36193 is added to blk_1073741860_1037 (size=35835) 2024-11-16T14:32:21,353 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=28.56 KB at sequenceid=1508 (bloomFilter=true), to=hdfs://localhost:43457/hbase/data/default/testReplayEditsWrittenIntoWAL/67a315f8951249f5de2391507bf27cca/.tmp/b/5126ac70253a49f4aeecaff44e4da77f 2024-11-16T14:32:21,361 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43457/hbase/data/default/testReplayEditsWrittenIntoWAL/67a315f8951249f5de2391507bf27cca/.tmp/a/b122abb60b364858875e3c83a80a0d07 as hdfs://localhost:43457/hbase/data/default/testReplayEditsWrittenIntoWAL/67a315f8951249f5de2391507bf27cca/a/b122abb60b364858875e3c83a80a0d07 2024-11-16T14:32:21,368 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43457/hbase/data/default/testReplayEditsWrittenIntoWAL/67a315f8951249f5de2391507bf27cca/a/b122abb60b364858875e3c83a80a0d07, entries=246, sequenceid=1508, filesize=19.6 K 2024-11-16T14:32:21,369 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43457/hbase/data/default/testReplayEditsWrittenIntoWAL/67a315f8951249f5de2391507bf27cca/.tmp/b/5126ac70253a49f4aeecaff44e4da77f as hdfs://localhost:43457/hbase/data/default/testReplayEditsWrittenIntoWAL/67a315f8951249f5de2391507bf27cca/b/5126ac70253a49f4aeecaff44e4da77f 2024-11-16T14:32:21,376 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43457/hbase/data/default/testReplayEditsWrittenIntoWAL/67a315f8951249f5de2391507bf27cca/b/5126ac70253a49f4aeecaff44e4da77f, entries=508, sequenceid=1508, filesize=35.0 K 2024-11-16T14:32:21,376 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~42.49 KB/43512, heapSize ~99.83 KB/102224, currentSize=0 B/0 for 67a315f8951249f5de2391507bf27cca in 86ms, sequenceid=1508, compaction requested=false; wal=null 2024-11-16T14:32:21,393 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-11-16T14:32:21,393 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 67a315f8951249f5de2391507bf27cca 3/3 column families, dataSize=42.49 KB heapSize=100.11 KB 2024-11-16T14:32:21,403 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43457/hbase/data/default/testReplayEditsWrittenIntoWAL/67a315f8951249f5de2391507bf27cca/.tmp/b/8c7e0fe7fbd44cfd8d53b27723c0d465 is 62, key is testReplayEditsWrittenIntoWAL/b:508/1731767536443/Put/seqid=0 2024-11-16T14:32:21,415 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36193 is added to blk_1073741861_1038 (size=35082) 2024-11-16T14:32:21,415 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33005 is added to blk_1073741861_1038 (size=35082) 2024-11-16T14:32:21,415 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43279 is added to blk_1073741861_1038 (size=35082) 2024-11-16T14:32:21,816 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=27.87 KB at sequenceid=2262 (bloomFilter=true), to=hdfs://localhost:43457/hbase/data/default/testReplayEditsWrittenIntoWAL/67a315f8951249f5de2391507bf27cca/.tmp/b/8c7e0fe7fbd44cfd8d53b27723c0d465 2024-11-16T14:32:21,840 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43457/hbase/data/default/testReplayEditsWrittenIntoWAL/67a315f8951249f5de2391507bf27cca/.tmp/c/d63714f7ec784ebea6c6ad4143fa960a is 62, key is testReplayEditsWrittenIntoWAL/c:100/1731767536482/Put/seqid=0 2024-11-16T14:32:21,846 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36193 is added to blk_1073741862_1039 (size=20825) 2024-11-16T14:32:21,847 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33005 is added to blk_1073741862_1039 (size=20825) 2024-11-16T14:32:21,847 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43279 is added to blk_1073741862_1039 (size=20825) 2024-11-16T14:32:21,847 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=14.63 KB at sequenceid=2262 (bloomFilter=true), to=hdfs://localhost:43457/hbase/data/default/testReplayEditsWrittenIntoWAL/67a315f8951249f5de2391507bf27cca/.tmp/c/d63714f7ec784ebea6c6ad4143fa960a 2024-11-16T14:32:21,854 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43457/hbase/data/default/testReplayEditsWrittenIntoWAL/67a315f8951249f5de2391507bf27cca/.tmp/b/8c7e0fe7fbd44cfd8d53b27723c0d465 as hdfs://localhost:43457/hbase/data/default/testReplayEditsWrittenIntoWAL/67a315f8951249f5de2391507bf27cca/b/8c7e0fe7fbd44cfd8d53b27723c0d465 2024-11-16T14:32:21,861 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43457/hbase/data/default/testReplayEditsWrittenIntoWAL/67a315f8951249f5de2391507bf27cca/b/8c7e0fe7fbd44cfd8d53b27723c0d465, entries=492, sequenceid=2262, filesize=34.3 K 2024-11-16T14:32:21,862 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43457/hbase/data/default/testReplayEditsWrittenIntoWAL/67a315f8951249f5de2391507bf27cca/.tmp/c/d63714f7ec784ebea6c6ad4143fa960a as hdfs://localhost:43457/hbase/data/default/testReplayEditsWrittenIntoWAL/67a315f8951249f5de2391507bf27cca/c/d63714f7ec784ebea6c6ad4143fa960a 2024-11-16T14:32:21,869 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43457/hbase/data/default/testReplayEditsWrittenIntoWAL/67a315f8951249f5de2391507bf27cca/c/d63714f7ec784ebea6c6ad4143fa960a, entries=262, sequenceid=2262, filesize=20.3 K 2024-11-16T14:32:21,869 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~42.49 KB/43512, heapSize ~99.83 KB/102224, currentSize=0 B/0 for 67a315f8951249f5de2391507bf27cca in 476ms, sequenceid=2262, compaction requested=false; wal=null 2024-11-16T14:32:21,881 WARN [Time-limited test {}] regionserver.HRegion(5722): No family for cell testReplayEditsWrittenIntoWAL/another family:testReplayEditsWrittenIntoWAL/1731767536535/Put/vlen=29/seqid=0 in region testReplayEditsWrittenIntoWAL,,1731767536217.67a315f8951249f5de2391507bf27cca. 2024-11-16T14:32:21,884 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 3001, skipped 1, firstSequenceIdInLog=1, maxSequenceIdInLog=3002, path=hdfs://localhost:43457/hbase/data/default/testReplayEditsWrittenIntoWAL/67a315f8951249f5de2391507bf27cca/recovered.edits/0000000000000003002 2024-11-16T14:32:21,884 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-11-16T14:32:21,885 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 67a315f8951249f5de2391507bf27cca 3/3 column families, dataSize=41.85 KB heapSize=98.89 KB 2024-11-16T14:32:21,895 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43457/hbase/data/default/testReplayEditsWrittenIntoWAL/67a315f8951249f5de2391507bf27cca/.tmp/c/26e025a2aa0e4131ba3063f37faf05b6 is 62, key is testReplayEditsWrittenIntoWAL/c:262/1731767536490/Put/seqid=0 2024-11-16T14:32:21,904 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36193 is added to blk_1073741863_1040 (size=50301) 2024-11-16T14:32:21,904 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43279 is added to blk_1073741863_1040 (size=50301) 2024-11-16T14:32:21,906 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33005 is added to blk_1073741863_1040 (size=50301) 2024-11-16T14:32:21,906 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=41.85 KB at sequenceid=3002 (bloomFilter=true), to=hdfs://localhost:43457/hbase/data/default/testReplayEditsWrittenIntoWAL/67a315f8951249f5de2391507bf27cca/.tmp/c/26e025a2aa0e4131ba3063f37faf05b6 2024-11-16T14:32:21,914 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 26e025a2aa0e4131ba3063f37faf05b6 2024-11-16T14:32:21,917 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43457/hbase/data/default/testReplayEditsWrittenIntoWAL/67a315f8951249f5de2391507bf27cca/.tmp/c/26e025a2aa0e4131ba3063f37faf05b6 as hdfs://localhost:43457/hbase/data/default/testReplayEditsWrittenIntoWAL/67a315f8951249f5de2391507bf27cca/c/26e025a2aa0e4131ba3063f37faf05b6 2024-11-16T14:32:21,923 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 26e025a2aa0e4131ba3063f37faf05b6 2024-11-16T14:32:21,923 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43457/hbase/data/default/testReplayEditsWrittenIntoWAL/67a315f8951249f5de2391507bf27cca/c/26e025a2aa0e4131ba3063f37faf05b6, entries=739, sequenceid=3002, filesize=49.1 K 2024-11-16T14:32:21,923 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~41.85 KB/42854, heapSize ~98.38 KB/100736, currentSize=0 B/0 for 67a315f8951249f5de2391507bf27cca in 39ms, sequenceid=3002, compaction requested=false; wal=null 2024-11-16T14:32:21,925 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:43457/hbase/data/default/testReplayEditsWrittenIntoWAL/67a315f8951249f5de2391507bf27cca/recovered.edits/0000000000000003002 2024-11-16T14:32:21,926 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 67a315f8951249f5de2391507bf27cca 2024-11-16T14:32:21,926 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 67a315f8951249f5de2391507bf27cca 2024-11-16T14:32:21,927 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenIntoWAL descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-16T14:32:21,929 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 67a315f8951249f5de2391507bf27cca 2024-11-16T14:32:21,932 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43457/hbase/data/default/testReplayEditsWrittenIntoWAL/67a315f8951249f5de2391507bf27cca/recovered.edits/3002.seqid, newMaxSeqId=3002, maxSeqId=1 2024-11-16T14:32:21,934 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 67a315f8951249f5de2391507bf27cca; next sequenceid=3003; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=204800, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63900990, jitterRate=-0.047801047563552856}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-16T14:32:21,934 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 67a315f8951249f5de2391507bf27cca: Writing region info on filesystem at 1731767540769Initializing all the Stores at 1731767540770 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731767540770Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731767540771 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731767540771Cleaning up temporary data from old regions at 1731767541926 (+1155 ms)Region opened successfully at 1731767541934 (+8 ms) 2024-11-16T14:32:22,003 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 67a315f8951249f5de2391507bf27cca, disabling compactions & flushes 2024-11-16T14:32:22,004 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1731767536217.67a315f8951249f5de2391507bf27cca. 2024-11-16T14:32:22,004 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1731767536217.67a315f8951249f5de2391507bf27cca. 2024-11-16T14:32:22,004 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1731767536217.67a315f8951249f5de2391507bf27cca. after waiting 0 ms 2024-11-16T14:32:22,004 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1731767536217.67a315f8951249f5de2391507bf27cca. 2024-11-16T14:32:22,006 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1731767536217.67a315f8951249f5de2391507bf27cca. 2024-11-16T14:32:22,006 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 67a315f8951249f5de2391507bf27cca: Waiting for close lock at 1731767542003Disabling compacts and flushes for region at 1731767542003Disabling writes for close at 1731767542004 (+1 ms)Writing region close event to WAL at 1731767542006 (+2 ms)Closed at 1731767542006 2024-11-16T14:32:22,011 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43279 is added to blk_1073741857_1034 (size=95) 2024-11-16T14:32:22,011 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731767536215/wal.1731767540749 not finished, retry = 0 2024-11-16T14:32:22,011 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33005 is added to blk_1073741857_1034 (size=95) 2024-11-16T14:32:22,011 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36193 is added to blk_1073741857_1034 (size=95) 2024-11-16T14:32:22,114 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-16T14:32:22,115 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1731767540749) 2024-11-16T14:32:22,126 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayValueCompression#testReplayEditsWrittenIntoWAL Thread=400 (was 386) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_372785318_22 at /127.0.0.1:59122 [Waiting for operation #24] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1319272049) connection to localhost/127.0.0.1:39925 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: IPC Client (1319272049) connection to localhost/127.0.0.1:43457 from jenkins.replay.wal.secondtime java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_372785318_22 at /127.0.0.1:45306 [Waiting for operation #15] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:39925 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/650842a5-0ab8-4683-35e8-38af23b31d67/cluster_174e5b33-598d-20b3-a770-4fb31341c285/data/data3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/650842a5-0ab8-4683-35e8-38af23b31d67/cluster_174e5b33-598d-20b3-a770-4fb31341c285/data/data4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_372785318_22 at /127.0.0.1:42646 [Waiting for operation #4] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-12-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:41505 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_372785318_22 at /127.0.0.1:42694 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-12-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: IPC Client (1319272049) connection to localhost/127.0.0.1:41505 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: AsyncFSWAL-12-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkins.replay.wal.secondtime@localhost:43457 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=865 (was 785) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=183 (was 170) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=4804 (was 4867) 2024-11-16T14:32:22,137 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayValueCompression#test2727 Thread=400, OpenFileDescriptor=865, MaxFileDescriptor=1048576, SystemLoadAverage=183, ProcessCount=11, AvailableMemoryMB=4803 2024-11-16T14:32:22,151 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-16T14:32:22,153 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-16T14:32:22,154 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-16T14:32:22,157 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-62525621, suffix=, logDir=hdfs://localhost:43457/hbase/WALs/hregion-62525621, archiveDir=hdfs://localhost:43457/hbase/oldWALs, maxLogs=32 2024-11-16T14:32:22,169 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-62525621/hregion-62525621.1731767542157, exclude list is [], retry=0 2024-11-16T14:32:22,172 DEBUG [AsyncFSWAL-14-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36193,DS-a30e140c-8532-4096-8cae-c8d713a3d4e3,DISK] 2024-11-16T14:32:22,173 DEBUG [AsyncFSWAL-14-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43279,DS-17ab9554-4509-49b3-918b-178692b9a473,DISK] 2024-11-16T14:32:22,173 DEBUG [AsyncFSWAL-14-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33005,DS-6a1dd528-c0be-4580-a53e-7113879f7f97,DISK] 2024-11-16T14:32:22,175 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-62525621/hregion-62525621.1731767542157 2024-11-16T14:32:22,176 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35677:35677),(127.0.0.1/127.0.0.1:36015:36015),(127.0.0.1/127.0.0.1:41179:41179)] 2024-11-16T14:32:22,176 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 901fab3b825512825637dfc4064d499f, NAME => 'test2727,,1731767542152.901fab3b825512825637dfc4064d499f.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='test2727', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43457/hbase 2024-11-16T14:32:22,185 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36193 is added to blk_1073741865_1042 (size=43) 2024-11-16T14:32:22,185 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33005 is added to blk_1073741865_1042 (size=43) 2024-11-16T14:32:22,186 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43279 is added to blk_1073741865_1042 (size=43) 2024-11-16T14:32:22,186 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated test2727,,1731767542152.901fab3b825512825637dfc4064d499f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T14:32:22,188 INFO [StoreOpener-901fab3b825512825637dfc4064d499f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 901fab3b825512825637dfc4064d499f 2024-11-16T14:32:22,190 INFO [StoreOpener-901fab3b825512825637dfc4064d499f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 901fab3b825512825637dfc4064d499f columnFamilyName a 2024-11-16T14:32:22,190 DEBUG [StoreOpener-901fab3b825512825637dfc4064d499f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:32:22,191 INFO [StoreOpener-901fab3b825512825637dfc4064d499f-1 {}] regionserver.HStore(327): Store=901fab3b825512825637dfc4064d499f/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T14:32:22,191 INFO [StoreOpener-901fab3b825512825637dfc4064d499f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 901fab3b825512825637dfc4064d499f 2024-11-16T14:32:22,193 INFO [StoreOpener-901fab3b825512825637dfc4064d499f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 901fab3b825512825637dfc4064d499f columnFamilyName b 2024-11-16T14:32:22,193 DEBUG [StoreOpener-901fab3b825512825637dfc4064d499f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:32:22,194 INFO [StoreOpener-901fab3b825512825637dfc4064d499f-1 {}] regionserver.HStore(327): Store=901fab3b825512825637dfc4064d499f/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T14:32:22,194 INFO [StoreOpener-901fab3b825512825637dfc4064d499f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 901fab3b825512825637dfc4064d499f 2024-11-16T14:32:22,196 INFO [StoreOpener-901fab3b825512825637dfc4064d499f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 901fab3b825512825637dfc4064d499f columnFamilyName c 2024-11-16T14:32:22,196 DEBUG [StoreOpener-901fab3b825512825637dfc4064d499f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:32:22,196 INFO [StoreOpener-901fab3b825512825637dfc4064d499f-1 {}] regionserver.HStore(327): Store=901fab3b825512825637dfc4064d499f/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T14:32:22,197 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 901fab3b825512825637dfc4064d499f 2024-11-16T14:32:22,197 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43457/hbase/data/default/test2727/901fab3b825512825637dfc4064d499f 2024-11-16T14:32:22,198 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43457/hbase/data/default/test2727/901fab3b825512825637dfc4064d499f 2024-11-16T14:32:22,199 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 901fab3b825512825637dfc4064d499f 2024-11-16T14:32:22,199 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 901fab3b825512825637dfc4064d499f 2024-11-16T14:32:22,199 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table test2727 descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-16T14:32:22,201 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 901fab3b825512825637dfc4064d499f 2024-11-16T14:32:22,203 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43457/hbase/data/default/test2727/901fab3b825512825637dfc4064d499f/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-16T14:32:22,204 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 901fab3b825512825637dfc4064d499f; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=58877301, jitterRate=-0.1226598471403122}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-16T14:32:22,205 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 901fab3b825512825637dfc4064d499f: Writing region info on filesystem at 1731767542187Initializing all the Stores at 1731767542187Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731767542187Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731767542188 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731767542188Cleaning up temporary data from old regions at 1731767542199 (+11 ms)Region opened successfully at 1731767542205 (+6 ms) 2024-11-16T14:32:22,205 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 901fab3b825512825637dfc4064d499f, disabling compactions & flushes 2024-11-16T14:32:22,205 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region test2727,,1731767542152.901fab3b825512825637dfc4064d499f. 2024-11-16T14:32:22,205 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on test2727,,1731767542152.901fab3b825512825637dfc4064d499f. 2024-11-16T14:32:22,205 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on test2727,,1731767542152.901fab3b825512825637dfc4064d499f. after waiting 0 ms 2024-11-16T14:32:22,205 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region test2727,,1731767542152.901fab3b825512825637dfc4064d499f. 2024-11-16T14:32:22,206 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed test2727,,1731767542152.901fab3b825512825637dfc4064d499f. 2024-11-16T14:32:22,206 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 901fab3b825512825637dfc4064d499f: Waiting for close lock at 1731767542205Disabling compacts and flushes for region at 1731767542205Disabling writes for close at 1731767542205Writing region close event to WAL at 1731767542206 (+1 ms)Closed at 1731767542206 2024-11-16T14:32:22,209 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36193 is added to blk_1073741864_1041 (size=95) 2024-11-16T14:32:22,209 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33005 is added to blk_1073741864_1041 (size=95) 2024-11-16T14:32:22,209 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43279 is added to blk_1073741864_1041 (size=95) 2024-11-16T14:32:22,214 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-16T14:32:22,214 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-62525621:(num 1731767542157) 2024-11-16T14:32:22,214 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-16T14:32:22,216 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:43457/hbase/WALs/test2727-manual,16010,1731767542150, archiveDir=hdfs://localhost:43457/hbase/oldWALs, maxLogs=32 2024-11-16T14:32:22,228 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/test2727-manual,16010,1731767542150/wal.1731767542217, exclude list is [], retry=0 2024-11-16T14:32:22,231 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36193,DS-a30e140c-8532-4096-8cae-c8d713a3d4e3,DISK] 2024-11-16T14:32:22,232 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33005,DS-6a1dd528-c0be-4580-a53e-7113879f7f97,DISK] 2024-11-16T14:32:22,232 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43279,DS-17ab9554-4509-49b3-918b-178692b9a473,DISK] 2024-11-16T14:32:22,234 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/test2727-manual,16010,1731767542150/wal.1731767542217 2024-11-16T14:32:22,234 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35677:35677),(127.0.0.1/127.0.0.1:41179:41179),(127.0.0.1/127.0.0.1:36015:36015)] 2024-11-16T14:32:22,371 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33005 is added to blk_1073741866_1043 (size=263359) 2024-11-16T14:32:22,372 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36193 is added to blk_1073741866_1043 (size=263359) 2024-11-16T14:32:22,372 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43279 is added to blk_1073741866_1043 (size=263359) 2024-11-16T14:32:22,389 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:43457/hbase/WALs/test2727-manual,16010,1731767542150/wal.1731767542217, size=257.2 K (263359bytes) 2024-11-16T14:32:22,389 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:43457/hbase/WALs/test2727-manual,16010,1731767542150/wal.1731767542217 2024-11-16T14:32:22,390 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:43457/hbase/WALs/test2727-manual,16010,1731767542150/wal.1731767542217 after 0ms 2024-11-16T14:32:22,393 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:43457/hbase/WALs/test2727-manual,16010,1731767542150/wal.1731767542217: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-16T14:32:22,395 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:43457/hbase/WALs/test2727-manual,16010,1731767542150/wal.1731767542217 took 6ms 2024-11-16T14:32:22,399 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal.1731767542217.temp 2024-11-16T14:32:22,401 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:43457/hbase/data/default/test2727/901fab3b825512825637dfc4064d499f/recovered.edits/0000000000000000001-wal.1731767542217.temp 2024-11-16T14:32:22,430 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-16T14:32:22,431 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-16T14:32:22,434 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-16T14:32:22,434 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-16T14:32:22,434 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-16T14:32:22,434 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-16T14:32:22,436 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenIntoWAL 2024-11-16T14:32:22,436 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenIntoWAL Metrics about Tables on a single HBase RegionServer 2024-11-16T14:32:22,458 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:43457/hbase/WALs/test2727-manual,16010,1731767542150/wal.1731767542217 so closing down 2024-11-16T14:32:22,458 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-16T14:32:22,459 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-16T14:32:22,461 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43279 is added to blk_1073741867_1044 (size=263359) 2024-11-16T14:32:22,462 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33005 is added to blk_1073741867_1044 (size=263359) 2024-11-16T14:32:22,462 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36193 is added to blk_1073741867_1044 (size=263359) 2024-11-16T14:32:22,463 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:43457/hbase/data/default/test2727/901fab3b825512825637dfc4064d499f/recovered.edits/0000000000000000001-wal.1731767542217.temp (wrote 3000 edits, skipped 0 edits in 39 ms) 2024-11-16T14:32:22,465 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:43457/hbase/data/default/test2727/901fab3b825512825637dfc4064d499f/recovered.edits/0000000000000000001-wal.1731767542217.temp to hdfs://localhost:43457/hbase/data/default/test2727/901fab3b825512825637dfc4064d499f/recovered.edits/0000000000000003000 2024-11-16T14:32:22,465 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 3000 edits across 1 Regions in 69 ms; skipped=0; WAL=hdfs://localhost:43457/hbase/WALs/test2727-manual,16010,1731767542150/wal.1731767542217, size=257.2 K, length=263359, corrupted=false, cancelled=false 2024-11-16T14:32:22,465 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:43457/hbase/WALs/test2727-manual,16010,1731767542150/wal.1731767542217, journal: Splitting hdfs://localhost:43457/hbase/WALs/test2727-manual,16010,1731767542150/wal.1731767542217, size=257.2 K (263359bytes) at 1731767542389Creating recovered edits writer path=hdfs://localhost:43457/hbase/data/default/test2727/901fab3b825512825637dfc4064d499f/recovered.edits/0000000000000000001-wal.1731767542217.temp at 1731767542401 (+12 ms)Split 1024 edits, skipped 0 edits. at 1731767542420 (+19 ms)Split 2048 edits, skipped 0 edits. at 1731767542442 (+22 ms)Finishing writing output for hdfs://localhost:43457/hbase/WALs/test2727-manual,16010,1731767542150/wal.1731767542217 so closing down at 1731767542458 (+16 ms)3 split writer threads finished at 1731767542459 (+1 ms)Closed recovered edits writer path=hdfs://localhost:43457/hbase/data/default/test2727/901fab3b825512825637dfc4064d499f/recovered.edits/0000000000000000001-wal.1731767542217.temp (wrote 3000 edits, skipped 0 edits in 39 ms) at 1731767542463 (+4 ms)Rename recovered edits hdfs://localhost:43457/hbase/data/default/test2727/901fab3b825512825637dfc4064d499f/recovered.edits/0000000000000000001-wal.1731767542217.temp to hdfs://localhost:43457/hbase/data/default/test2727/901fab3b825512825637dfc4064d499f/recovered.edits/0000000000000003000 at 1731767542465 (+2 ms)Processed 3000 edits across 1 Regions in 69 ms; skipped=0; WAL=hdfs://localhost:43457/hbase/WALs/test2727-manual,16010,1731767542150/wal.1731767542217, size=257.2 K, length=263359, corrupted=false, cancelled=false at 1731767542465 2024-11-16T14:32:22,468 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:43457/hbase/WALs/test2727-manual,16010,1731767542150/wal.1731767542217 to hdfs://localhost:43457/hbase/oldWALs/wal.1731767542217 2024-11-16T14:32:22,469 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:43457/hbase/data/default/test2727/901fab3b825512825637dfc4064d499f/recovered.edits/0000000000000003000 2024-11-16T14:32:22,469 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-16T14:32:22,472 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:43457/hbase/WALs/test2727-manual,16010,1731767542150, archiveDir=hdfs://localhost:43457/hbase/oldWALs, maxLogs=32 2024-11-16T14:32:22,485 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/test2727-manual,16010,1731767542150/wal.1731767542472, exclude list is [], retry=0 2024-11-16T14:32:22,489 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33005,DS-6a1dd528-c0be-4580-a53e-7113879f7f97,DISK] 2024-11-16T14:32:22,489 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43279,DS-17ab9554-4509-49b3-918b-178692b9a473,DISK] 2024-11-16T14:32:22,490 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36193,DS-a30e140c-8532-4096-8cae-c8d713a3d4e3,DISK] 2024-11-16T14:32:22,493 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/test2727-manual,16010,1731767542150/wal.1731767542472 2024-11-16T14:32:22,493 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41179:41179),(127.0.0.1/127.0.0.1:36015:36015),(127.0.0.1/127.0.0.1:35677:35677)] 2024-11-16T14:32:22,648 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33005 is added to blk_1073741868_1045 (size=263486) 2024-11-16T14:32:22,648 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36193 is added to blk_1073741868_1045 (size=263486) 2024-11-16T14:32:22,649 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43279 is added to blk_1073741868_1045 (size=263486) 2024-11-16T14:32:22,670 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:43457/hbase/WALs/test2727-manual,16010,1731767542150/wal.1731767542472, size=257.3 K (263486bytes) 2024-11-16T14:32:22,670 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:43457/hbase/WALs/test2727-manual,16010,1731767542150/wal.1731767542472 2024-11-16T14:32:22,671 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:43457/hbase/WALs/test2727-manual,16010,1731767542150/wal.1731767542472 after 1ms 2024-11-16T14:32:22,674 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:43457/hbase/WALs/test2727-manual,16010,1731767542150/wal.1731767542472: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-16T14:32:22,675 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:43457/hbase/WALs/test2727-manual,16010,1731767542150/wal.1731767542472 took 5ms 2024-11-16T14:32:22,679 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000003001-wal.1731767542472.temp 2024-11-16T14:32:22,681 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:43457/hbase/data/default/test2727/901fab3b825512825637dfc4064d499f/recovered.edits/0000000000000003001-wal.1731767542472.temp 2024-11-16T14:32:22,729 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:43457/hbase/WALs/test2727-manual,16010,1731767542150/wal.1731767542472 so closing down 2024-11-16T14:32:22,729 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-16T14:32:22,729 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-16T14:32:22,731 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43279 is added to blk_1073741869_1046 (size=263486) 2024-11-16T14:32:22,732 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36193 is added to blk_1073741869_1046 (size=263486) 2024-11-16T14:32:22,732 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33005 is added to blk_1073741869_1046 (size=263486) 2024-11-16T14:32:22,733 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:43457/hbase/data/default/test2727/901fab3b825512825637dfc4064d499f/recovered.edits/0000000000000003001-wal.1731767542472.temp (wrote 3000 edits, skipped 0 edits in 34 ms) 2024-11-16T14:32:22,734 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:43457/hbase/data/default/test2727/901fab3b825512825637dfc4064d499f/recovered.edits/0000000000000003001-wal.1731767542472.temp to hdfs://localhost:43457/hbase/data/default/test2727/901fab3b825512825637dfc4064d499f/recovered.edits/0000000000000006000 2024-11-16T14:32:22,734 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 3000 edits across 1 Regions in 59 ms; skipped=0; WAL=hdfs://localhost:43457/hbase/WALs/test2727-manual,16010,1731767542150/wal.1731767542472, size=257.3 K, length=263486, corrupted=false, cancelled=false 2024-11-16T14:32:22,734 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:43457/hbase/WALs/test2727-manual,16010,1731767542150/wal.1731767542472, journal: Splitting hdfs://localhost:43457/hbase/WALs/test2727-manual,16010,1731767542150/wal.1731767542472, size=257.3 K (263486bytes) at 1731767542670Creating recovered edits writer path=hdfs://localhost:43457/hbase/data/default/test2727/901fab3b825512825637dfc4064d499f/recovered.edits/0000000000000003001-wal.1731767542472.temp at 1731767542681 (+11 ms)Split 1024 edits, skipped 0 edits. at 1731767542693 (+12 ms)Split 2048 edits, skipped 0 edits. at 1731767542711 (+18 ms)Finishing writing output for hdfs://localhost:43457/hbase/WALs/test2727-manual,16010,1731767542150/wal.1731767542472 so closing down at 1731767542729 (+18 ms)3 split writer threads finished at 1731767542729Closed recovered edits writer path=hdfs://localhost:43457/hbase/data/default/test2727/901fab3b825512825637dfc4064d499f/recovered.edits/0000000000000003001-wal.1731767542472.temp (wrote 3000 edits, skipped 0 edits in 34 ms) at 1731767542733 (+4 ms)Rename recovered edits hdfs://localhost:43457/hbase/data/default/test2727/901fab3b825512825637dfc4064d499f/recovered.edits/0000000000000003001-wal.1731767542472.temp to hdfs://localhost:43457/hbase/data/default/test2727/901fab3b825512825637dfc4064d499f/recovered.edits/0000000000000006000 at 1731767542734 (+1 ms)Processed 3000 edits across 1 Regions in 59 ms; skipped=0; WAL=hdfs://localhost:43457/hbase/WALs/test2727-manual,16010,1731767542150/wal.1731767542472, size=257.3 K, length=263486, corrupted=false, cancelled=false at 1731767542734 2024-11-16T14:32:22,736 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:43457/hbase/WALs/test2727-manual,16010,1731767542150/wal.1731767542472 to hdfs://localhost:43457/hbase/oldWALs/wal.1731767542472 2024-11-16T14:32:22,737 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:43457/hbase/data/default/test2727/901fab3b825512825637dfc4064d499f/recovered.edits/0000000000000006000 2024-11-16T14:32:22,738 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-16T14:32:22,740 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:43457/hbase/WALs/test2727-manual,16010,1731767542150, archiveDir=hdfs://localhost:43457/hbase/oldWALs, maxLogs=32 2024-11-16T14:32:22,753 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/test2727-manual,16010,1731767542150/wal.1731767542740, exclude list is [], retry=0 2024-11-16T14:32:22,756 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33005,DS-6a1dd528-c0be-4580-a53e-7113879f7f97,DISK] 2024-11-16T14:32:22,757 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36193,DS-a30e140c-8532-4096-8cae-c8d713a3d4e3,DISK] 2024-11-16T14:32:22,757 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43279,DS-17ab9554-4509-49b3-918b-178692b9a473,DISK] 2024-11-16T14:32:22,759 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/test2727-manual,16010,1731767542150/wal.1731767542740 2024-11-16T14:32:22,760 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41179:41179),(127.0.0.1/127.0.0.1:35677:35677),(127.0.0.1/127.0.0.1:36015:36015)] 2024-11-16T14:32:22,760 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 901fab3b825512825637dfc4064d499f, NAME => 'test2727,,1731767542152.901fab3b825512825637dfc4064d499f.', STARTKEY => '', ENDKEY => ''} 2024-11-16T14:32:22,760 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated test2727,,1731767542152.901fab3b825512825637dfc4064d499f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T14:32:22,760 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 901fab3b825512825637dfc4064d499f 2024-11-16T14:32:22,760 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 901fab3b825512825637dfc4064d499f 2024-11-16T14:32:22,763 INFO [StoreOpener-901fab3b825512825637dfc4064d499f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 901fab3b825512825637dfc4064d499f 2024-11-16T14:32:22,764 INFO [StoreOpener-901fab3b825512825637dfc4064d499f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 901fab3b825512825637dfc4064d499f columnFamilyName a 2024-11-16T14:32:22,764 DEBUG [StoreOpener-901fab3b825512825637dfc4064d499f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:32:22,764 INFO [StoreOpener-901fab3b825512825637dfc4064d499f-1 {}] regionserver.HStore(327): Store=901fab3b825512825637dfc4064d499f/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T14:32:22,765 INFO [StoreOpener-901fab3b825512825637dfc4064d499f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 901fab3b825512825637dfc4064d499f 2024-11-16T14:32:22,766 INFO [StoreOpener-901fab3b825512825637dfc4064d499f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 901fab3b825512825637dfc4064d499f columnFamilyName b 2024-11-16T14:32:22,766 DEBUG [StoreOpener-901fab3b825512825637dfc4064d499f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:32:22,766 INFO [StoreOpener-901fab3b825512825637dfc4064d499f-1 {}] regionserver.HStore(327): Store=901fab3b825512825637dfc4064d499f/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T14:32:22,766 INFO [StoreOpener-901fab3b825512825637dfc4064d499f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 901fab3b825512825637dfc4064d499f 2024-11-16T14:32:22,767 INFO [StoreOpener-901fab3b825512825637dfc4064d499f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 901fab3b825512825637dfc4064d499f columnFamilyName c 2024-11-16T14:32:22,768 DEBUG [StoreOpener-901fab3b825512825637dfc4064d499f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:32:22,768 INFO [StoreOpener-901fab3b825512825637dfc4064d499f-1 {}] regionserver.HStore(327): Store=901fab3b825512825637dfc4064d499f/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T14:32:22,768 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 901fab3b825512825637dfc4064d499f 2024-11-16T14:32:22,769 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43457/hbase/data/default/test2727/901fab3b825512825637dfc4064d499f 2024-11-16T14:32:22,772 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 2 recovered edits file(s) under hdfs://localhost:43457/hbase/data/default/test2727/901fab3b825512825637dfc4064d499f 2024-11-16T14:32:22,772 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:43457/hbase/data/default/test2727/901fab3b825512825637dfc4064d499f/recovered.edits/0000000000000003000 2024-11-16T14:32:22,775 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:43457/hbase/data/default/test2727/901fab3b825512825637dfc4064d499f/recovered.edits/0000000000000003000: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-16T14:32:22,832 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 3000, skipped 0, firstSequenceIdInLog=1, maxSequenceIdInLog=3000, path=hdfs://localhost:43457/hbase/data/default/test2727/901fab3b825512825637dfc4064d499f/recovered.edits/0000000000000003000 2024-11-16T14:32:22,833 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:43457/hbase/data/default/test2727/901fab3b825512825637dfc4064d499f/recovered.edits/0000000000000006000 2024-11-16T14:32:22,836 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:43457/hbase/data/default/test2727/901fab3b825512825637dfc4064d499f/recovered.edits/0000000000000006000: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-16T14:32:22,880 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 3000, skipped 0, firstSequenceIdInLog=3001, maxSequenceIdInLog=6000, path=hdfs://localhost:43457/hbase/data/default/test2727/901fab3b825512825637dfc4064d499f/recovered.edits/0000000000000006000 2024-11-16T14:32:22,880 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 901fab3b825512825637dfc4064d499f 3/3 column families, dataSize=215.51 KB heapSize=657 KB 2024-11-16T14:32:22,900 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43457/hbase/data/default/test2727/901fab3b825512825637dfc4064d499f/.tmp/a/bc039008498f400bb4295c5f90d77fef is 41, key is test2727/a:100/1731767542500/Put/seqid=0 2024-11-16T14:32:22,905 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36193 is added to blk_1073741871_1048 (size=84227) 2024-11-16T14:32:22,905 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43279 is added to blk_1073741871_1048 (size=84227) 2024-11-16T14:32:22,906 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33005 is added to blk_1073741871_1048 (size=84227) 2024-11-16T14:32:22,907 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=71.84 KB at sequenceid=6000 (bloomFilter=true), to=hdfs://localhost:43457/hbase/data/default/test2727/901fab3b825512825637dfc4064d499f/.tmp/a/bc039008498f400bb4295c5f90d77fef 2024-11-16T14:32:22,935 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43457/hbase/data/default/test2727/901fab3b825512825637dfc4064d499f/.tmp/b/e6db396ccc624817ba35bb83e54e418c is 41, key is test2727/b:100/1731767542548/Put/seqid=0 2024-11-16T14:32:22,939 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36193 is added to blk_1073741872_1049 (size=84609) 2024-11-16T14:32:22,939 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43279 is added to blk_1073741872_1049 (size=84609) 2024-11-16T14:32:22,940 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33005 is added to blk_1073741872_1049 (size=84609) 2024-11-16T14:32:22,940 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=71.84 KB at sequenceid=6000 (bloomFilter=true), to=hdfs://localhost:43457/hbase/data/default/test2727/901fab3b825512825637dfc4064d499f/.tmp/b/e6db396ccc624817ba35bb83e54e418c 2024-11-16T14:32:22,966 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43457/hbase/data/default/test2727/901fab3b825512825637dfc4064d499f/.tmp/c/0975fc9f0fb64af898dcf150a69db3e0 is 41, key is test2727/c:100/1731767542601/Put/seqid=0 2024-11-16T14:32:22,973 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43279 is added to blk_1073741873_1050 (size=84609) 2024-11-16T14:32:22,973 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33005 is added to blk_1073741873_1050 (size=84609) 2024-11-16T14:32:22,974 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36193 is added to blk_1073741873_1050 (size=84609) 2024-11-16T14:32:22,974 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=71.84 KB at sequenceid=6000 (bloomFilter=true), to=hdfs://localhost:43457/hbase/data/default/test2727/901fab3b825512825637dfc4064d499f/.tmp/c/0975fc9f0fb64af898dcf150a69db3e0 2024-11-16T14:32:22,983 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43457/hbase/data/default/test2727/901fab3b825512825637dfc4064d499f/.tmp/a/bc039008498f400bb4295c5f90d77fef as hdfs://localhost:43457/hbase/data/default/test2727/901fab3b825512825637dfc4064d499f/a/bc039008498f400bb4295c5f90d77fef 2024-11-16T14:32:22,991 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43457/hbase/data/default/test2727/901fab3b825512825637dfc4064d499f/a/bc039008498f400bb4295c5f90d77fef, entries=2000, sequenceid=6000, filesize=82.3 K 2024-11-16T14:32:22,992 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43457/hbase/data/default/test2727/901fab3b825512825637dfc4064d499f/.tmp/b/e6db396ccc624817ba35bb83e54e418c as hdfs://localhost:43457/hbase/data/default/test2727/901fab3b825512825637dfc4064d499f/b/e6db396ccc624817ba35bb83e54e418c 2024-11-16T14:32:23,001 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43457/hbase/data/default/test2727/901fab3b825512825637dfc4064d499f/b/e6db396ccc624817ba35bb83e54e418c, entries=2000, sequenceid=6000, filesize=82.6 K 2024-11-16T14:32:23,002 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43457/hbase/data/default/test2727/901fab3b825512825637dfc4064d499f/.tmp/c/0975fc9f0fb64af898dcf150a69db3e0 as hdfs://localhost:43457/hbase/data/default/test2727/901fab3b825512825637dfc4064d499f/c/0975fc9f0fb64af898dcf150a69db3e0 2024-11-16T14:32:23,011 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43457/hbase/data/default/test2727/901fab3b825512825637dfc4064d499f/c/0975fc9f0fb64af898dcf150a69db3e0, entries=2000, sequenceid=6000, filesize=82.6 K 2024-11-16T14:32:23,011 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~215.51 KB/220680, heapSize ~656.95 KB/672720, currentSize=0 B/0 for 901fab3b825512825637dfc4064d499f in 131ms, sequenceid=6000, compaction requested=false; wal=null 2024-11-16T14:32:23,012 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:43457/hbase/data/default/test2727/901fab3b825512825637dfc4064d499f/recovered.edits/0000000000000003000 2024-11-16T14:32:23,013 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:43457/hbase/data/default/test2727/901fab3b825512825637dfc4064d499f/recovered.edits/0000000000000006000 2024-11-16T14:32:23,015 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 901fab3b825512825637dfc4064d499f 2024-11-16T14:32:23,015 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 901fab3b825512825637dfc4064d499f 2024-11-16T14:32:23,015 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table test2727 descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-16T14:32:23,017 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 901fab3b825512825637dfc4064d499f 2024-11-16T14:32:23,020 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43457/hbase/data/default/test2727/901fab3b825512825637dfc4064d499f/recovered.edits/6000.seqid, newMaxSeqId=6000, maxSeqId=1 2024-11-16T14:32:23,021 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 901fab3b825512825637dfc4064d499f; next sequenceid=6001; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61421781, jitterRate=-0.08474414050579071}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-16T14:32:23,023 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 901fab3b825512825637dfc4064d499f: Writing region info on filesystem at 1731767542760Initializing all the Stores at 1731767542761 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731767542762 (+1 ms)Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731767542762Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731767542762Obtaining lock to block concurrent updates at 1731767542880 (+118 ms)Preparing flush snapshotting stores in 901fab3b825512825637dfc4064d499f at 1731767542880Finished memstore snapshotting test2727,,1731767542152.901fab3b825512825637dfc4064d499f., syncing WAL and waiting on mvcc, flushsize=dataSize=220680, getHeapSize=672720, getOffHeapSize=0, getCellsCount=6000 at 1731767542881 (+1 ms)Flushing stores of test2727,,1731767542152.901fab3b825512825637dfc4064d499f. at 1731767542881Flushing 901fab3b825512825637dfc4064d499f/a: creating writer at 1731767542881Flushing 901fab3b825512825637dfc4064d499f/a: appending metadata at 1731767542899 (+18 ms)Flushing 901fab3b825512825637dfc4064d499f/a: closing flushed file at 1731767542899Flushing 901fab3b825512825637dfc4064d499f/b: creating writer at 1731767542914 (+15 ms)Flushing 901fab3b825512825637dfc4064d499f/b: appending metadata at 1731767542933 (+19 ms)Flushing 901fab3b825512825637dfc4064d499f/b: closing flushed file at 1731767542933Flushing 901fab3b825512825637dfc4064d499f/c: creating writer at 1731767542947 (+14 ms)Flushing 901fab3b825512825637dfc4064d499f/c: appending metadata at 1731767542966 (+19 ms)Flushing 901fab3b825512825637dfc4064d499f/c: closing flushed file at 1731767542966Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@431b1ee: reopening flushed file at 1731767542981 (+15 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6fc846e: reopening flushed file at 1731767542991 (+10 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@13a1547d: reopening flushed file at 1731767543001 (+10 ms)Finished flush of dataSize ~215.51 KB/220680, heapSize ~656.95 KB/672720, currentSize=0 B/0 for 901fab3b825512825637dfc4064d499f in 131ms, sequenceid=6000, compaction requested=false; wal=null at 1731767543011 (+10 ms)Cleaning up temporary data from old regions at 1731767543015 (+4 ms)Region opened successfully at 1731767543023 (+8 ms) 2024-11-16T14:32:23,025 DEBUG [Time-limited test {}] wal.AbstractTestWALReplay(320): region.getOpenSeqNum(): 6001, wal3.id: 0 2024-11-16T14:32:23,025 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 901fab3b825512825637dfc4064d499f, disabling compactions & flushes 2024-11-16T14:32:23,025 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region test2727,,1731767542152.901fab3b825512825637dfc4064d499f. 2024-11-16T14:32:23,025 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on test2727,,1731767542152.901fab3b825512825637dfc4064d499f. 2024-11-16T14:32:23,025 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on test2727,,1731767542152.901fab3b825512825637dfc4064d499f. after waiting 0 ms 2024-11-16T14:32:23,025 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region test2727,,1731767542152.901fab3b825512825637dfc4064d499f. 2024-11-16T14:32:23,027 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed test2727,,1731767542152.901fab3b825512825637dfc4064d499f. 2024-11-16T14:32:23,027 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 901fab3b825512825637dfc4064d499f: Waiting for close lock at 1731767543025Disabling compacts and flushes for region at 1731767543025Disabling writes for close at 1731767543025Writing region close event to WAL at 1731767543027 (+2 ms)Closed at 1731767543027 2024-11-16T14:32:23,031 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43279 is added to blk_1073741870_1047 (size=95) 2024-11-16T14:32:23,031 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33005 is added to blk_1073741870_1047 (size=95) 2024-11-16T14:32:23,032 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36193 is added to blk_1073741870_1047 (size=95) 2024-11-16T14:32:23,034 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-16T14:32:23,034 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1731767542740) 2024-11-16T14:32:23,045 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayValueCompression#test2727 Thread=402 (was 400) Potentially hanging thread: AsyncFSWAL-14-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1995475656_22 at /127.0.0.1:57404 [Waiting for operation #11] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1995475656_22 at /127.0.0.1:37466 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1995475656_22 at /127.0.0.1:42694 [Waiting for operation #9] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-14-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-14-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=927 (was 865) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=183 (was 183), ProcessCount=11 (was 11), AvailableMemoryMB=4594 (was 4803) 2024-11-16T14:32:23,055 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayValueCompression#testSequentialEditLogSeqNum Thread=402, OpenFileDescriptor=927, MaxFileDescriptor=1048576, SystemLoadAverage=183, ProcessCount=11, AvailableMemoryMB=4593 2024-11-16T14:32:23,071 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-16T14:32:23,078 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:43457/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1731767543070, archiveDir=hdfs://localhost:43457/hbase/oldWALs, maxLogs=32 2024-11-16T14:32:23,078 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731767543078 2024-11-16T14:32:23,086 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testsequentialeditlogseqnum-manual,16010,1731767543070/wal.1731767543078 2024-11-16T14:32:23,087 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new MockWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41179:41179),(127.0.0.1/127.0.0.1:35677:35677),(127.0.0.1/127.0.0.1:36015:36015)] 2024-11-16T14:32:23,089 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 69637fb77184487f10224ccc5341827c, NAME => 'testSequentialEditLogSeqNum,,1731767543072.69637fb77184487f10224ccc5341827c.', STARTKEY => '', ENDKEY => ''} 2024-11-16T14:32:23,089 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testSequentialEditLogSeqNum,,1731767543072.69637fb77184487f10224ccc5341827c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T14:32:23,089 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 69637fb77184487f10224ccc5341827c 2024-11-16T14:32:23,089 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 69637fb77184487f10224ccc5341827c 2024-11-16T14:32:23,090 WARN [Time-limited test {}] regionserver.HRegionFileSystem(836): hdfs://localhost:43457/hbase/data/default/testSequentialEditLogSeqNum/69637fb77184487f10224ccc5341827c doesn't exist for region: 69637fb77184487f10224ccc5341827c on table testSequentialEditLogSeqNum 2024-11-16T14:32:23,090 WARN [Time-limited test {}] regionserver.HRegionFileSystem(854): .regioninfo file not found for region: 69637fb77184487f10224ccc5341827c on table testSequentialEditLogSeqNum 2024-11-16T14:32:23,098 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33005 is added to blk_1073741875_1052 (size=62) 2024-11-16T14:32:23,098 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43279 is added to blk_1073741875_1052 (size=62) 2024-11-16T14:32:23,099 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36193 is added to blk_1073741875_1052 (size=62) 2024-11-16T14:32:23,101 INFO [StoreOpener-69637fb77184487f10224ccc5341827c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 69637fb77184487f10224ccc5341827c 2024-11-16T14:32:23,103 INFO [StoreOpener-69637fb77184487f10224ccc5341827c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 69637fb77184487f10224ccc5341827c columnFamilyName a 2024-11-16T14:32:23,103 DEBUG [StoreOpener-69637fb77184487f10224ccc5341827c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:32:23,104 INFO [StoreOpener-69637fb77184487f10224ccc5341827c-1 {}] regionserver.HStore(327): Store=69637fb77184487f10224ccc5341827c/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T14:32:23,104 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 69637fb77184487f10224ccc5341827c 2024-11-16T14:32:23,105 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43457/hbase/data/default/testSequentialEditLogSeqNum/69637fb77184487f10224ccc5341827c 2024-11-16T14:32:23,105 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43457/hbase/data/default/testSequentialEditLogSeqNum/69637fb77184487f10224ccc5341827c 2024-11-16T14:32:23,106 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 69637fb77184487f10224ccc5341827c 2024-11-16T14:32:23,106 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 69637fb77184487f10224ccc5341827c 2024-11-16T14:32:23,108 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 69637fb77184487f10224ccc5341827c 2024-11-16T14:32:23,112 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43457/hbase/data/default/testSequentialEditLogSeqNum/69637fb77184487f10224ccc5341827c/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-16T14:32:23,113 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 69637fb77184487f10224ccc5341827c; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59570036, jitterRate=-0.11233729124069214}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-16T14:32:23,115 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 69637fb77184487f10224ccc5341827c: Writing region info on filesystem at 1731767543089Initializing all the Stores at 1731767543101 (+12 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731767543101Cleaning up temporary data from old regions at 1731767543106 (+5 ms)Region opened successfully at 1731767543115 (+9 ms) 2024-11-16T14:32:23,130 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 69637fb77184487f10224ccc5341827c 1/1 column families, dataSize=770 B heapSize=1.73 KB 2024-11-16T14:32:23,159 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43457/hbase/data/default/testSequentialEditLogSeqNum/69637fb77184487f10224ccc5341827c/.tmp/a/ab004efa12e5455da26d9571b4b07be0 is 81, key is testSequentialEditLogSeqNum/a:x0/1731767543115/Put/seqid=0 2024-11-16T14:32:23,167 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33005 is added to blk_1073741876_1053 (size=5833) 2024-11-16T14:32:23,167 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43279 is added to blk_1073741876_1053 (size=5833) 2024-11-16T14:32:23,168 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36193 is added to blk_1073741876_1053 (size=5833) 2024-11-16T14:32:23,168 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=770 B at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:43457/hbase/data/default/testSequentialEditLogSeqNum/69637fb77184487f10224ccc5341827c/.tmp/a/ab004efa12e5455da26d9571b4b07be0 2024-11-16T14:32:23,177 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43457/hbase/data/default/testSequentialEditLogSeqNum/69637fb77184487f10224ccc5341827c/.tmp/a/ab004efa12e5455da26d9571b4b07be0 as hdfs://localhost:43457/hbase/data/default/testSequentialEditLogSeqNum/69637fb77184487f10224ccc5341827c/a/ab004efa12e5455da26d9571b4b07be0 2024-11-16T14:32:23,185 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43457/hbase/data/default/testSequentialEditLogSeqNum/69637fb77184487f10224ccc5341827c/a/ab004efa12e5455da26d9571b4b07be0, entries=10, sequenceid=13, filesize=5.7 K 2024-11-16T14:32:23,187 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~770 B/770, heapSize ~1.72 KB/1760, currentSize=0 B/0 for 69637fb77184487f10224ccc5341827c in 57ms, sequenceid=13, compaction requested=false 2024-11-16T14:32:23,187 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 69637fb77184487f10224ccc5341827c: 2024-11-16T14:32:23,195 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-16T14:32:23,195 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-16T14:32:23,195 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-16T14:32:23,195 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-16T14:32:23,196 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-16T14:32:23,198 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43279 is added to blk_1073741874_1051 (size=1843) 2024-11-16T14:32:23,198 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36193 is added to blk_1073741874_1051 (size=1843) 2024-11-16T14:32:23,198 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33005 is added to blk_1073741874_1051 (size=1843) 2024-11-16T14:32:23,216 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:43457/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1731767543070/wal.1731767543078, size=1.8 K (1843bytes) 2024-11-16T14:32:23,216 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:43457/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1731767543070/wal.1731767543078 2024-11-16T14:32:23,216 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:43457/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1731767543070/wal.1731767543078 after 0ms 2024-11-16T14:32:23,219 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:43457/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1731767543070/wal.1731767543078: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-16T14:32:23,219 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:43457/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1731767543070/wal.1731767543078 took 4ms 2024-11-16T14:32:23,223 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:43457/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1731767543070/wal.1731767543078 so closing down 2024-11-16T14:32:23,223 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-16T14:32:23,224 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1731767543078.temp 2024-11-16T14:32:23,225 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:43457/hbase/data/default/testSequentialEditLogSeqNum/69637fb77184487f10224ccc5341827c/recovered.edits/0000000000000000003-wal.1731767543078.temp 2024-11-16T14:32:23,226 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-16T14:32:23,234 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43279 is added to blk_1073741877_1054 (size=1477) 2024-11-16T14:32:23,234 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36193 is added to blk_1073741877_1054 (size=1477) 2024-11-16T14:32:23,235 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33005 is added to blk_1073741877_1054 (size=1477) 2024-11-16T14:32:23,235 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:43457/hbase/data/default/testSequentialEditLogSeqNum/69637fb77184487f10224ccc5341827c/recovered.edits/0000000000000000003-wal.1731767543078.temp (wrote 15 edits, skipped 0 edits in 0 ms) 2024-11-16T14:32:23,237 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:43457/hbase/data/default/testSequentialEditLogSeqNum/69637fb77184487f10224ccc5341827c/recovered.edits/0000000000000000003-wal.1731767543078.temp to hdfs://localhost:43457/hbase/data/default/testSequentialEditLogSeqNum/69637fb77184487f10224ccc5341827c/recovered.edits/0000000000000000020 2024-11-16T14:32:23,237 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 17 edits across 1 Regions in 17 ms; skipped=2; WAL=hdfs://localhost:43457/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1731767543070/wal.1731767543078, size=1.8 K, length=1843, corrupted=false, cancelled=false 2024-11-16T14:32:23,237 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:43457/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1731767543070/wal.1731767543078, journal: Splitting hdfs://localhost:43457/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1731767543070/wal.1731767543078, size=1.8 K (1843bytes) at 1731767543216Finishing writing output for hdfs://localhost:43457/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1731767543070/wal.1731767543078 so closing down at 1731767543223 (+7 ms)Creating recovered edits writer path=hdfs://localhost:43457/hbase/data/default/testSequentialEditLogSeqNum/69637fb77184487f10224ccc5341827c/recovered.edits/0000000000000000003-wal.1731767543078.temp at 1731767543225 (+2 ms)3 split writer threads finished at 1731767543226 (+1 ms)Closed recovered edits writer path=hdfs://localhost:43457/hbase/data/default/testSequentialEditLogSeqNum/69637fb77184487f10224ccc5341827c/recovered.edits/0000000000000000003-wal.1731767543078.temp (wrote 15 edits, skipped 0 edits in 0 ms) at 1731767543235 (+9 ms)Rename recovered edits hdfs://localhost:43457/hbase/data/default/testSequentialEditLogSeqNum/69637fb77184487f10224ccc5341827c/recovered.edits/0000000000000000003-wal.1731767543078.temp to hdfs://localhost:43457/hbase/data/default/testSequentialEditLogSeqNum/69637fb77184487f10224ccc5341827c/recovered.edits/0000000000000000020 at 1731767543237 (+2 ms)Processed 17 edits across 1 Regions in 17 ms; skipped=2; WAL=hdfs://localhost:43457/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1731767543070/wal.1731767543078, size=1.8 K, length=1843, corrupted=false, cancelled=false at 1731767543237 2024-11-16T14:32:23,257 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayValueCompression#testSequentialEditLogSeqNum Thread=409 (was 402) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/650842a5-0ab8-4683-35e8-38af23b31d67/cluster_174e5b33-598d-20b3-a770-4fb31341c285/data/data1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: sync.0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1995475656_22 at /127.0.0.1:57404 [Waiting for operation #13] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1995475656_22 at /127.0.0.1:37466 [Waiting for operation #11] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: sync.2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/650842a5-0ab8-4683-35e8-38af23b31d67/cluster_174e5b33-598d-20b3-a770-4fb31341c285/data/data2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=963 (was 927) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=183 (was 183), ProcessCount=11 (was 11), AvailableMemoryMB=4565 (was 4593) 2024-11-16T14:32:23,271 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayValueCompression#testRegionMadeOfBulkLoadedFilesOnly Thread=409, OpenFileDescriptor=963, MaxFileDescriptor=1048576, SystemLoadAverage=183, ProcessCount=11, AvailableMemoryMB=4561 2024-11-16T14:32:23,288 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-16T14:32:23,319 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-16T14:32:23,349 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-16T14:32:23,353 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-13853425, suffix=, logDir=hdfs://localhost:43457/hbase/WALs/hregion-13853425, archiveDir=hdfs://localhost:43457/hbase/oldWALs, maxLogs=32 2024-11-16T14:32:23,382 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-13853425/hregion-13853425.1731767543360, exclude list is [], retry=0 2024-11-16T14:32:23,385 DEBUG [AsyncFSWAL-17-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43279,DS-17ab9554-4509-49b3-918b-178692b9a473,DISK] 2024-11-16T14:32:23,385 DEBUG [AsyncFSWAL-17-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36193,DS-a30e140c-8532-4096-8cae-c8d713a3d4e3,DISK] 2024-11-16T14:32:23,386 DEBUG [AsyncFSWAL-17-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33005,DS-6a1dd528-c0be-4580-a53e-7113879f7f97,DISK] 2024-11-16T14:32:23,389 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-13853425/hregion-13853425.1731767543360 2024-11-16T14:32:23,391 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35677:35677),(127.0.0.1/127.0.0.1:36015:36015),(127.0.0.1/127.0.0.1:41179:41179)] 2024-11-16T14:32:23,391 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 891f0e783575ea009660942d99448221, NAME => 'testRegionMadeOfBulkLoadedFilesOnly,,1731767543289.891f0e783575ea009660942d99448221.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testRegionMadeOfBulkLoadedFilesOnly', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43457/hbase 2024-11-16T14:32:23,405 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43279 is added to blk_1073741879_1056 (size=70) 2024-11-16T14:32:23,406 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33005 is added to blk_1073741879_1056 (size=70) 2024-11-16T14:32:23,406 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36193 is added to blk_1073741879_1056 (size=70) 2024-11-16T14:32:23,406 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testRegionMadeOfBulkLoadedFilesOnly,,1731767543289.891f0e783575ea009660942d99448221.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T14:32:23,408 INFO [StoreOpener-891f0e783575ea009660942d99448221-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 891f0e783575ea009660942d99448221 2024-11-16T14:32:23,411 INFO [StoreOpener-891f0e783575ea009660942d99448221-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 891f0e783575ea009660942d99448221 columnFamilyName a 2024-11-16T14:32:23,411 DEBUG [StoreOpener-891f0e783575ea009660942d99448221-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:32:23,412 INFO [StoreOpener-891f0e783575ea009660942d99448221-1 {}] regionserver.HStore(327): Store=891f0e783575ea009660942d99448221/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T14:32:23,412 INFO [StoreOpener-891f0e783575ea009660942d99448221-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 891f0e783575ea009660942d99448221 2024-11-16T14:32:23,414 INFO [StoreOpener-891f0e783575ea009660942d99448221-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 891f0e783575ea009660942d99448221 columnFamilyName b 2024-11-16T14:32:23,414 DEBUG [StoreOpener-891f0e783575ea009660942d99448221-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:32:23,415 INFO [StoreOpener-891f0e783575ea009660942d99448221-1 {}] regionserver.HStore(327): Store=891f0e783575ea009660942d99448221/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T14:32:23,415 INFO [StoreOpener-891f0e783575ea009660942d99448221-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 891f0e783575ea009660942d99448221 2024-11-16T14:32:23,416 INFO [StoreOpener-891f0e783575ea009660942d99448221-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 891f0e783575ea009660942d99448221 columnFamilyName c 2024-11-16T14:32:23,417 DEBUG [StoreOpener-891f0e783575ea009660942d99448221-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:32:23,417 INFO [StoreOpener-891f0e783575ea009660942d99448221-1 {}] regionserver.HStore(327): Store=891f0e783575ea009660942d99448221/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T14:32:23,417 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 891f0e783575ea009660942d99448221 2024-11-16T14:32:23,418 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43457/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/891f0e783575ea009660942d99448221 2024-11-16T14:32:23,418 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43457/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/891f0e783575ea009660942d99448221 2024-11-16T14:32:23,419 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 891f0e783575ea009660942d99448221 2024-11-16T14:32:23,420 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 891f0e783575ea009660942d99448221 2024-11-16T14:32:23,420 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRegionMadeOfBulkLoadedFilesOnly descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-16T14:32:23,422 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 891f0e783575ea009660942d99448221 2024-11-16T14:32:23,424 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43457/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/891f0e783575ea009660942d99448221/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-16T14:32:23,425 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 891f0e783575ea009660942d99448221; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72794066, jitterRate=0.08471611142158508}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-16T14:32:23,426 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 891f0e783575ea009660942d99448221: Writing region info on filesystem at 1731767543406Initializing all the Stores at 1731767543407 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731767543407Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731767543408 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731767543408Cleaning up temporary data from old regions at 1731767543420 (+12 ms)Region opened successfully at 1731767543426 (+6 ms) 2024-11-16T14:32:23,426 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 891f0e783575ea009660942d99448221, disabling compactions & flushes 2024-11-16T14:32:23,426 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testRegionMadeOfBulkLoadedFilesOnly,,1731767543289.891f0e783575ea009660942d99448221. 2024-11-16T14:32:23,426 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testRegionMadeOfBulkLoadedFilesOnly,,1731767543289.891f0e783575ea009660942d99448221. 2024-11-16T14:32:23,427 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testRegionMadeOfBulkLoadedFilesOnly,,1731767543289.891f0e783575ea009660942d99448221. after waiting 0 ms 2024-11-16T14:32:23,427 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testRegionMadeOfBulkLoadedFilesOnly,,1731767543289.891f0e783575ea009660942d99448221. 2024-11-16T14:32:23,427 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testRegionMadeOfBulkLoadedFilesOnly,,1731767543289.891f0e783575ea009660942d99448221. 2024-11-16T14:32:23,427 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 891f0e783575ea009660942d99448221: Waiting for close lock at 1731767543426Disabling compacts and flushes for region at 1731767543426Disabling writes for close at 1731767543427 (+1 ms)Writing region close event to WAL at 1731767543427Closed at 1731767543427 2024-11-16T14:32:23,430 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33005 is added to blk_1073741878_1055 (size=95) 2024-11-16T14:32:23,430 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43279 is added to blk_1073741878_1055 (size=95) 2024-11-16T14:32:23,431 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36193 is added to blk_1073741878_1055 (size=95) 2024-11-16T14:32:23,434 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-16T14:32:23,434 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-13853425:(num 1731767543360) 2024-11-16T14:32:23,434 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-16T14:32:23,436 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:43457/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731767543287, archiveDir=hdfs://localhost:43457/hbase/oldWALs, maxLogs=32 2024-11-16T14:32:23,455 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731767543287/wal.1731767543437, exclude list is [], retry=0 2024-11-16T14:32:23,458 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43279,DS-17ab9554-4509-49b3-918b-178692b9a473,DISK] 2024-11-16T14:32:23,458 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33005,DS-6a1dd528-c0be-4580-a53e-7113879f7f97,DISK] 2024-11-16T14:32:23,459 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36193,DS-a30e140c-8532-4096-8cae-c8d713a3d4e3,DISK] 2024-11-16T14:32:23,461 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731767543287/wal.1731767543437 2024-11-16T14:32:23,461 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36015:36015),(127.0.0.1/127.0.0.1:41179:41179),(127.0.0.1/127.0.0.1:35677:35677)] 2024-11-16T14:32:23,461 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 891f0e783575ea009660942d99448221, NAME => 'testRegionMadeOfBulkLoadedFilesOnly,,1731767543289.891f0e783575ea009660942d99448221.', STARTKEY => '', ENDKEY => ''} 2024-11-16T14:32:23,461 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testRegionMadeOfBulkLoadedFilesOnly,,1731767543289.891f0e783575ea009660942d99448221.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T14:32:23,462 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 891f0e783575ea009660942d99448221 2024-11-16T14:32:23,462 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 891f0e783575ea009660942d99448221 2024-11-16T14:32:23,463 INFO [StoreOpener-891f0e783575ea009660942d99448221-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 891f0e783575ea009660942d99448221 2024-11-16T14:32:23,465 INFO [StoreOpener-891f0e783575ea009660942d99448221-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 891f0e783575ea009660942d99448221 columnFamilyName a 2024-11-16T14:32:23,465 DEBUG [StoreOpener-891f0e783575ea009660942d99448221-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:32:23,465 INFO [StoreOpener-891f0e783575ea009660942d99448221-1 {}] regionserver.HStore(327): Store=891f0e783575ea009660942d99448221/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T14:32:23,466 INFO [StoreOpener-891f0e783575ea009660942d99448221-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 891f0e783575ea009660942d99448221 2024-11-16T14:32:23,467 INFO [StoreOpener-891f0e783575ea009660942d99448221-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 891f0e783575ea009660942d99448221 columnFamilyName b 2024-11-16T14:32:23,467 DEBUG [StoreOpener-891f0e783575ea009660942d99448221-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:32:23,467 INFO [StoreOpener-891f0e783575ea009660942d99448221-1 {}] regionserver.HStore(327): Store=891f0e783575ea009660942d99448221/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T14:32:23,468 INFO [StoreOpener-891f0e783575ea009660942d99448221-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 891f0e783575ea009660942d99448221 2024-11-16T14:32:23,469 INFO [StoreOpener-891f0e783575ea009660942d99448221-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 891f0e783575ea009660942d99448221 columnFamilyName c 2024-11-16T14:32:23,469 DEBUG [StoreOpener-891f0e783575ea009660942d99448221-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:32:23,470 INFO [StoreOpener-891f0e783575ea009660942d99448221-1 {}] regionserver.HStore(327): Store=891f0e783575ea009660942d99448221/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T14:32:23,470 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 891f0e783575ea009660942d99448221 2024-11-16T14:32:23,471 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43457/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/891f0e783575ea009660942d99448221 2024-11-16T14:32:23,472 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43457/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/891f0e783575ea009660942d99448221 2024-11-16T14:32:23,474 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 891f0e783575ea009660942d99448221 2024-11-16T14:32:23,474 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 891f0e783575ea009660942d99448221 2024-11-16T14:32:23,474 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRegionMadeOfBulkLoadedFilesOnly descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-16T14:32:23,476 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 891f0e783575ea009660942d99448221 2024-11-16T14:32:23,477 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 891f0e783575ea009660942d99448221; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61669172, jitterRate=-0.08105772733688354}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-16T14:32:23,479 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 891f0e783575ea009660942d99448221: Writing region info on filesystem at 1731767543462Initializing all the Stores at 1731767543463 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731767543463Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731767543463Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731767543463Cleaning up temporary data from old regions at 1731767543474 (+11 ms)Region opened successfully at 1731767543479 (+5 ms) 2024-11-16T14:32:23,484 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43457/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile is 28, key is \x0D/a:a/1731767543482/Put/seqid=0 2024-11-16T14:32:23,490 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36193 is added to blk_1073741881_1058 (size=4826) 2024-11-16T14:32:23,491 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33005 is added to blk_1073741881_1058 (size=4826) 2024-11-16T14:32:23,491 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43279 is added to blk_1073741881_1058 (size=4826) 2024-11-16T14:32:23,492 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:43457/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile for inclusion in 891f0e783575ea009660942d99448221/a 2024-11-16T14:32:23,502 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first= last=z 2024-11-16T14:32:23,502 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-11-16T14:32:23,502 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 891f0e783575ea009660942d99448221: 2024-11-16T14:32:23,504 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43457/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile as hdfs://localhost:43457/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/891f0e783575ea009660942d99448221/a/b43436abf9d048af9da24d06a6bfbaef_SeqId_3_ 2024-11-16T14:32:23,506 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:43457/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile into 891f0e783575ea009660942d99448221/a as hdfs://localhost:43457/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/891f0e783575ea009660942d99448221/a/b43436abf9d048af9da24d06a6bfbaef_SeqId_3_ - updating store file list. 2024-11-16T14:32:23,513 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for b43436abf9d048af9da24d06a6bfbaef_SeqId_3_: NONE, but ROW specified in column family configuration 2024-11-16T14:32:23,513 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:43457/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/891f0e783575ea009660942d99448221/a/b43436abf9d048af9da24d06a6bfbaef_SeqId_3_ into 891f0e783575ea009660942d99448221/a 2024-11-16T14:32:23,513 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:43457/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile into 891f0e783575ea009660942d99448221/a (new location: hdfs://localhost:43457/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/891f0e783575ea009660942d99448221/a/b43436abf9d048af9da24d06a6bfbaef_SeqId_3_) 2024-11-16T14:32:23,556 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:43457/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731767543287/wal.1731767543437, size=0 (0bytes) 2024-11-16T14:32:23,556 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:43457/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731767543287/wal.1731767543437 might be still open, length is 0 2024-11-16T14:32:23,556 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:43457/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731767543287/wal.1731767543437 2024-11-16T14:32:23,556 WARN [IPC Server handler 3 on default port 43457 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731767543287/wal.1731767543437 has not been closed. Lease recovery is in progress. RecoveryId = 1059 for block blk_1073741880_1057 2024-11-16T14:32:23,556 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:43457/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731767543287/wal.1731767543437 after 0ms 2024-11-16T14:32:26,260 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1995475656_22 at /127.0.0.1:37564 [Receiving block BP-865919460-172.17.0.2-1731767527819:blk_1073741880_1057] {}] datanode.DataXceiver(331): 127.0.0.1:43279:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:37564 dst: /127.0.0.1:43279 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:43279 remote=/127.0.0.1:37564]. Total timeout mills is 60000, 57261 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:32:26,260 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1995475656_22 at /127.0.0.1:59870 [Receiving block BP-865919460-172.17.0.2-1731767527819:blk_1073741880_1057] {}] datanode.DataXceiver(331): 127.0.0.1:33005:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:59870 dst: /127.0.0.1:33005 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:32:26,260 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1995475656_22 at /127.0.0.1:57520 [Receiving block BP-865919460-172.17.0.2-1731767527819:blk_1073741880_1057] {}] datanode.DataXceiver(331): 127.0.0.1:36193:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:57520 dst: /127.0.0.1:36193 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:32:26,264 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43279 is added to blk_1073741880_1059 (size=472) 2024-11-16T14:32:26,265 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33005 is added to blk_1073741880_1059 (size=472) 2024-11-16T14:32:27,558 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:43457/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731767543287/wal.1731767543437 after 4002ms 2024-11-16T14:32:27,567 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:43457/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731767543287/wal.1731767543437: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-16T14:32:27,568 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:43457/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731767543287/wal.1731767543437 took 4012ms 2024-11-16T14:32:27,570 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:43457/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731767543287/wal.1731767543437; continuing. 2024-11-16T14:32:27,570 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:43457/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731767543287/wal.1731767543437 so closing down 2024-11-16T14:32:27,570 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-16T14:32:27,572 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000005-wal.1731767543437.temp 2024-11-16T14:32:27,574 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:43457/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/891f0e783575ea009660942d99448221/recovered.edits/0000000000000000005-wal.1731767543437.temp 2024-11-16T14:32:27,574 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-16T14:32:27,580 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33005 is added to blk_1073741882_1060 (size=259) 2024-11-16T14:32:27,581 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43279 is added to blk_1073741882_1060 (size=259) 2024-11-16T14:32:27,581 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36193 is added to blk_1073741882_1060 (size=259) 2024-11-16T14:32:27,581 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:43457/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/891f0e783575ea009660942d99448221/recovered.edits/0000000000000000005-wal.1731767543437.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-11-16T14:32:27,583 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:43457/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/891f0e783575ea009660942d99448221/recovered.edits/0000000000000000005-wal.1731767543437.temp to hdfs://localhost:43457/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/891f0e783575ea009660942d99448221/recovered.edits/0000000000000000005 2024-11-16T14:32:27,583 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 2 edits across 1 Regions in 15 ms; skipped=1; WAL=hdfs://localhost:43457/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731767543287/wal.1731767543437, size=0, length=0, corrupted=false, cancelled=false 2024-11-16T14:32:27,583 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:43457/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731767543287/wal.1731767543437, journal: Splitting hdfs://localhost:43457/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731767543287/wal.1731767543437, size=0 (0bytes) at 1731767543556Finishing writing output for hdfs://localhost:43457/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731767543287/wal.1731767543437 so closing down at 1731767547570 (+4014 ms)Creating recovered edits writer path=hdfs://localhost:43457/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/891f0e783575ea009660942d99448221/recovered.edits/0000000000000000005-wal.1731767543437.temp at 1731767547574 (+4 ms)3 split writer threads finished at 1731767547574Closed recovered edits writer path=hdfs://localhost:43457/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/891f0e783575ea009660942d99448221/recovered.edits/0000000000000000005-wal.1731767543437.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1731767547581 (+7 ms)Rename recovered edits hdfs://localhost:43457/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/891f0e783575ea009660942d99448221/recovered.edits/0000000000000000005-wal.1731767543437.temp to hdfs://localhost:43457/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/891f0e783575ea009660942d99448221/recovered.edits/0000000000000000005 at 1731767547583 (+2 ms)Processed 2 edits across 1 Regions in 15 ms; skipped=1; WAL=hdfs://localhost:43457/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731767543287/wal.1731767543437, size=0, length=0, corrupted=false, cancelled=false at 1731767547583 2024-11-16T14:32:27,585 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:43457/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731767543287/wal.1731767543437 to hdfs://localhost:43457/hbase/oldWALs/wal.1731767543437 2024-11-16T14:32:27,586 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:43457/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/891f0e783575ea009660942d99448221/recovered.edits/0000000000000000005 2024-11-16T14:32:27,586 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-16T14:32:27,588 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:43457/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731767543287, archiveDir=hdfs://localhost:43457/hbase/oldWALs, maxLogs=32 2024-11-16T14:32:27,603 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731767543287/wal.1731767547588, exclude list is [], retry=0 2024-11-16T14:32:27,605 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36193,DS-a30e140c-8532-4096-8cae-c8d713a3d4e3,DISK] 2024-11-16T14:32:27,606 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33005,DS-6a1dd528-c0be-4580-a53e-7113879f7f97,DISK] 2024-11-16T14:32:27,606 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43279,DS-17ab9554-4509-49b3-918b-178692b9a473,DISK] 2024-11-16T14:32:27,608 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731767543287/wal.1731767547588 2024-11-16T14:32:27,608 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35677:35677),(127.0.0.1/127.0.0.1:41179:41179),(127.0.0.1/127.0.0.1:36015:36015)] 2024-11-16T14:32:27,608 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 891f0e783575ea009660942d99448221, NAME => 'testRegionMadeOfBulkLoadedFilesOnly,,1731767543289.891f0e783575ea009660942d99448221.', STARTKEY => '', ENDKEY => ''} 2024-11-16T14:32:27,608 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testRegionMadeOfBulkLoadedFilesOnly,,1731767543289.891f0e783575ea009660942d99448221.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T14:32:27,608 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 891f0e783575ea009660942d99448221 2024-11-16T14:32:27,609 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 891f0e783575ea009660942d99448221 2024-11-16T14:32:27,610 INFO [StoreOpener-891f0e783575ea009660942d99448221-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 891f0e783575ea009660942d99448221 2024-11-16T14:32:27,611 INFO [StoreOpener-891f0e783575ea009660942d99448221-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 891f0e783575ea009660942d99448221 columnFamilyName a 2024-11-16T14:32:27,611 DEBUG [StoreOpener-891f0e783575ea009660942d99448221-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:32:27,617 DEBUG [StoreFileOpener-891f0e783575ea009660942d99448221-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for b43436abf9d048af9da24d06a6bfbaef_SeqId_3_: NONE, but ROW specified in column family configuration 2024-11-16T14:32:27,617 DEBUG [StoreOpener-891f0e783575ea009660942d99448221-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:43457/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/891f0e783575ea009660942d99448221/a/b43436abf9d048af9da24d06a6bfbaef_SeqId_3_ 2024-11-16T14:32:27,618 INFO [StoreOpener-891f0e783575ea009660942d99448221-1 {}] regionserver.HStore(327): Store=891f0e783575ea009660942d99448221/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T14:32:27,618 INFO [StoreOpener-891f0e783575ea009660942d99448221-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 891f0e783575ea009660942d99448221 2024-11-16T14:32:27,619 INFO [StoreOpener-891f0e783575ea009660942d99448221-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 891f0e783575ea009660942d99448221 columnFamilyName b 2024-11-16T14:32:27,619 DEBUG [StoreOpener-891f0e783575ea009660942d99448221-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:32:27,620 INFO [StoreOpener-891f0e783575ea009660942d99448221-1 {}] regionserver.HStore(327): Store=891f0e783575ea009660942d99448221/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T14:32:27,620 INFO [StoreOpener-891f0e783575ea009660942d99448221-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 891f0e783575ea009660942d99448221 2024-11-16T14:32:27,621 INFO [StoreOpener-891f0e783575ea009660942d99448221-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 891f0e783575ea009660942d99448221 columnFamilyName c 2024-11-16T14:32:27,621 DEBUG [StoreOpener-891f0e783575ea009660942d99448221-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:32:27,621 INFO [StoreOpener-891f0e783575ea009660942d99448221-1 {}] regionserver.HStore(327): Store=891f0e783575ea009660942d99448221/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T14:32:27,622 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 891f0e783575ea009660942d99448221 2024-11-16T14:32:27,622 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43457/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/891f0e783575ea009660942d99448221 2024-11-16T14:32:27,624 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:43457/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/891f0e783575ea009660942d99448221 2024-11-16T14:32:27,624 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:43457/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/891f0e783575ea009660942d99448221/recovered.edits/0000000000000000005 2024-11-16T14:32:27,626 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:43457/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/891f0e783575ea009660942d99448221/recovered.edits/0000000000000000005: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-16T14:32:27,627 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 1, skipped 0, firstSequenceIdInLog=5, maxSequenceIdInLog=5, path=hdfs://localhost:43457/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/891f0e783575ea009660942d99448221/recovered.edits/0000000000000000005 2024-11-16T14:32:27,627 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 891f0e783575ea009660942d99448221 3/3 column families, dataSize=58 B heapSize=904 B 2024-11-16T14:32:27,642 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43457/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/891f0e783575ea009660942d99448221/.tmp/a/b736858d62c34522a089bf85d23fc253 is 62, key is testRegionMadeOfBulkLoadedFilesOnly/a:a/1731767543519/Put/seqid=0 2024-11-16T14:32:27,648 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33005 is added to blk_1073741884_1062 (size=5149) 2024-11-16T14:32:27,648 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36193 is added to blk_1073741884_1062 (size=5149) 2024-11-16T14:32:27,648 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43279 is added to blk_1073741884_1062 (size=5149) 2024-11-16T14:32:27,649 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58 B at sequenceid=5 (bloomFilter=true), to=hdfs://localhost:43457/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/891f0e783575ea009660942d99448221/.tmp/a/b736858d62c34522a089bf85d23fc253 2024-11-16T14:32:27,655 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43457/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/891f0e783575ea009660942d99448221/.tmp/a/b736858d62c34522a089bf85d23fc253 as hdfs://localhost:43457/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/891f0e783575ea009660942d99448221/a/b736858d62c34522a089bf85d23fc253 2024-11-16T14:32:27,662 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43457/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/891f0e783575ea009660942d99448221/a/b736858d62c34522a089bf85d23fc253, entries=1, sequenceid=5, filesize=5.0 K 2024-11-16T14:32:27,662 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~58 B/58, heapSize ~376 B/376, currentSize=0 B/0 for 891f0e783575ea009660942d99448221 in 35ms, sequenceid=5, compaction requested=false; wal=null 2024-11-16T14:32:27,663 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:43457/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/891f0e783575ea009660942d99448221/recovered.edits/0000000000000000005 2024-11-16T14:32:27,665 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 891f0e783575ea009660942d99448221 2024-11-16T14:32:27,665 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 891f0e783575ea009660942d99448221 2024-11-16T14:32:27,666 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRegionMadeOfBulkLoadedFilesOnly descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-16T14:32:27,669 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 891f0e783575ea009660942d99448221 2024-11-16T14:32:27,671 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43457/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/891f0e783575ea009660942d99448221/recovered.edits/5.seqid, newMaxSeqId=5, maxSeqId=1 2024-11-16T14:32:27,673 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 891f0e783575ea009660942d99448221; next sequenceid=6; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61259309, jitterRate=-0.08716516196727753}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-16T14:32:27,673 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 891f0e783575ea009660942d99448221: Writing region info on filesystem at 1731767547609Initializing all the Stores at 1731767547610 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731767547610Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731767547610Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731767547610Obtaining lock to block concurrent updates at 1731767547627 (+17 ms)Preparing flush snapshotting stores in 891f0e783575ea009660942d99448221 at 1731767547627Finished memstore snapshotting testRegionMadeOfBulkLoadedFilesOnly,,1731767543289.891f0e783575ea009660942d99448221., syncing WAL and waiting on mvcc, flushsize=dataSize=58, getHeapSize=856, getOffHeapSize=0, getCellsCount=1 at 1731767547627Flushing stores of testRegionMadeOfBulkLoadedFilesOnly,,1731767543289.891f0e783575ea009660942d99448221. at 1731767547627Flushing 891f0e783575ea009660942d99448221/a: creating writer at 1731767547627Flushing 891f0e783575ea009660942d99448221/a: appending metadata at 1731767547642 (+15 ms)Flushing 891f0e783575ea009660942d99448221/a: closing flushed file at 1731767547642Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@75bbb1cb: reopening flushed file at 1731767547654 (+12 ms)Finished flush of dataSize ~58 B/58, heapSize ~376 B/376, currentSize=0 B/0 for 891f0e783575ea009660942d99448221 in 35ms, sequenceid=5, compaction requested=false; wal=null at 1731767547662 (+8 ms)Cleaning up temporary data from old regions at 1731767547665 (+3 ms)Region opened successfully at 1731767547673 (+8 ms) 2024-11-16T14:32:27,676 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 891f0e783575ea009660942d99448221, disabling compactions & flushes 2024-11-16T14:32:27,677 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testRegionMadeOfBulkLoadedFilesOnly,,1731767543289.891f0e783575ea009660942d99448221. 2024-11-16T14:32:27,677 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testRegionMadeOfBulkLoadedFilesOnly,,1731767543289.891f0e783575ea009660942d99448221. 2024-11-16T14:32:27,677 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testRegionMadeOfBulkLoadedFilesOnly,,1731767543289.891f0e783575ea009660942d99448221. after waiting 0 ms 2024-11-16T14:32:27,677 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testRegionMadeOfBulkLoadedFilesOnly,,1731767543289.891f0e783575ea009660942d99448221. 2024-11-16T14:32:27,678 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testRegionMadeOfBulkLoadedFilesOnly,,1731767543289.891f0e783575ea009660942d99448221. 2024-11-16T14:32:27,678 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 891f0e783575ea009660942d99448221: Waiting for close lock at 1731767547676Disabling compacts and flushes for region at 1731767547676Disabling writes for close at 1731767547677 (+1 ms)Writing region close event to WAL at 1731767547677Closed at 1731767547678 (+1 ms) 2024-11-16T14:32:27,680 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43279 is added to blk_1073741883_1061 (size=95) 2024-11-16T14:32:27,681 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36193 is added to blk_1073741883_1061 (size=95) 2024-11-16T14:32:27,681 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33005 is added to blk_1073741883_1061 (size=95) 2024-11-16T14:32:27,683 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-16T14:32:27,683 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1731767547588) 2024-11-16T14:32:27,696 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayValueCompression#testRegionMadeOfBulkLoadedFilesOnly Thread=411 (was 409) Potentially hanging thread: AsyncFSWAL-17-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1818902652_22 at /127.0.0.1:59894 [Waiting for operation #5] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-17-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1818902652_22 at /127.0.0.1:57558 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-17-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1818902652_22 at /127.0.0.1:37584 [Waiting for operation #5] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1319272049) connection to localhost/127.0.0.1:43457 from jenkinstestRegionMadeOfBulkLoadedFilesOnly java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: LeaseRenewer:jenkinstestRegionMadeOfBulkLoadedFilesOnly@localhost:43457 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1023 (was 963) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=168 (was 183), ProcessCount=11 (was 11), AvailableMemoryMB=4519 (was 4561) 2024-11-16T14:32:27,709 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayValueCompression#testReplayEditsAfterRegionMovedWithMultiCF Thread=411, OpenFileDescriptor=1023, MaxFileDescriptor=1048576, SystemLoadAverage=168, ProcessCount=11, AvailableMemoryMB=4518 2024-11-16T14:32:27,723 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-16T14:32:27,727 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-16T14:32:27,731 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 04e93f2b5fad,34365,1731767532092 2024-11-16T14:32:27,733 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@157a4fe5 2024-11-16T14:32:27,734 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-16T14:32:27,736 INFO [HMaster-EventLoopGroup-2-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56534, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-16T14:32:27,740 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34365 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testReplayEditsAfterRegionMovedWithMultiCF', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-16T14:32:27,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34365 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF 2024-11-16T14:32:27,748 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_PRE_OPERATION 2024-11-16T14:32:27,750 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34365 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testReplayEditsAfterRegionMovedWithMultiCF" procId is: 4 2024-11-16T14:32:27,750 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:32:27,752 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-16T14:32:27,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34365 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-16T14:32:27,763 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36193 is added to blk_1073741885_1063 (size=694) 2024-11-16T14:32:27,763 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43279 is added to blk_1073741885_1063 (size=694) 2024-11-16T14:32:27,763 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33005 is added to blk_1073741885_1063 (size=694) 2024-11-16T14:32:27,766 INFO [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 9cc76f2c81b25eccf90da7ca97d0721e, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1731767547738.9cc76f2c81b25eccf90da7ca97d0721e.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsAfterRegionMovedWithMultiCF', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43457/user/jenkins/test-data/0d13f5ec-97ce-91d4-7824-c5174956db56 2024-11-16T14:32:27,777 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33005 is added to blk_1073741886_1064 (size=77) 2024-11-16T14:32:27,777 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43279 is added to blk_1073741886_1064 (size=77) 2024-11-16T14:32:27,777 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36193 is added to blk_1073741886_1064 (size=77) 2024-11-16T14:32:27,782 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1731767547738.9cc76f2c81b25eccf90da7ca97d0721e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T14:32:27,782 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1722): Closing 9cc76f2c81b25eccf90da7ca97d0721e, disabling compactions & flushes 2024-11-16T14:32:27,782 INFO [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1731767547738.9cc76f2c81b25eccf90da7ca97d0721e. 2024-11-16T14:32:27,782 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731767547738.9cc76f2c81b25eccf90da7ca97d0721e. 2024-11-16T14:32:27,782 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731767547738.9cc76f2c81b25eccf90da7ca97d0721e. after waiting 0 ms 2024-11-16T14:32:27,782 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1731767547738.9cc76f2c81b25eccf90da7ca97d0721e. 2024-11-16T14:32:27,782 INFO [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1731767547738.9cc76f2c81b25eccf90da7ca97d0721e. 2024-11-16T14:32:27,782 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1676): Region close journal for 9cc76f2c81b25eccf90da7ca97d0721e: Waiting for close lock at 1731767547782Disabling compacts and flushes for region at 1731767547782Disabling writes for close at 1731767547782Writing region close event to WAL at 1731767547782Closed at 1731767547782 2024-11-16T14:32:27,784 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_ADD_TO_META 2024-11-16T14:32:27,790 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testReplayEditsAfterRegionMovedWithMultiCF,,1731767547738.9cc76f2c81b25eccf90da7ca97d0721e.","families":{"info":[{"qualifier":"regioninfo","vlen":76,"tag":[],"timestamp":"1731767547784"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731767547784"}]},"ts":"1731767547784"} 2024-11-16T14:32:27,794 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-16T14:32:27,795 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-16T14:32:27,798 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testReplayEditsAfterRegionMovedWithMultiCF","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731767547796"}]},"ts":"1731767547796"} 2024-11-16T14:32:27,801 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testReplayEditsAfterRegionMovedWithMultiCF, state=ENABLING in hbase:meta 2024-11-16T14:32:27,802 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {04e93f2b5fad=0} racks are {/default-rack=0} 2024-11-16T14:32:27,803 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-16T14:32:27,803 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-16T14:32:27,803 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-11-16T14:32:27,803 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-16T14:32:27,803 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-16T14:32:27,803 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-11-16T14:32:27,803 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-16T14:32:27,804 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-16T14:32:27,804 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-11-16T14:32:27,804 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-16T14:32:27,805 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=9cc76f2c81b25eccf90da7ca97d0721e, ASSIGN}] 2024-11-16T14:32:27,807 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=9cc76f2c81b25eccf90da7ca97d0721e, ASSIGN 2024-11-16T14:32:27,809 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=9cc76f2c81b25eccf90da7ca97d0721e, ASSIGN; state=OFFLINE, location=04e93f2b5fad,45119,1731767533018; forceNewPlan=false, retain=false 2024-11-16T14:32:27,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34365 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-16T14:32:27,963 INFO [04e93f2b5fad:34365 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-16T14:32:27,965 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=9cc76f2c81b25eccf90da7ca97d0721e, regionState=OPENING, regionLocation=04e93f2b5fad,45119,1731767533018 2024-11-16T14:32:27,973 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=9cc76f2c81b25eccf90da7ca97d0721e, ASSIGN because future has completed 2024-11-16T14:32:27,974 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9cc76f2c81b25eccf90da7ca97d0721e, server=04e93f2b5fad,45119,1731767533018}] 2024-11-16T14:32:28,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34365 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-16T14:32:28,128 DEBUG [RSProcedureDispatcher-pool-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-16T14:32:28,130 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47945, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-16T14:32:28,137 INFO [RS_OPEN_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1731767547738.9cc76f2c81b25eccf90da7ca97d0721e. 2024-11-16T14:32:28,137 DEBUG [RS_OPEN_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 9cc76f2c81b25eccf90da7ca97d0721e, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1731767547738.9cc76f2c81b25eccf90da7ca97d0721e.', STARTKEY => '', ENDKEY => ''} 2024-11-16T14:32:28,137 DEBUG [RS_OPEN_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF 9cc76f2c81b25eccf90da7ca97d0721e 2024-11-16T14:32:28,138 DEBUG [RS_OPEN_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1731767547738.9cc76f2c81b25eccf90da7ca97d0721e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T14:32:28,138 DEBUG [RS_OPEN_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 9cc76f2c81b25eccf90da7ca97d0721e 2024-11-16T14:32:28,138 DEBUG [RS_OPEN_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 9cc76f2c81b25eccf90da7ca97d0721e 2024-11-16T14:32:28,140 INFO [StoreOpener-9cc76f2c81b25eccf90da7ca97d0721e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 9cc76f2c81b25eccf90da7ca97d0721e 2024-11-16T14:32:28,142 INFO [StoreOpener-9cc76f2c81b25eccf90da7ca97d0721e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9cc76f2c81b25eccf90da7ca97d0721e columnFamilyName cf1 2024-11-16T14:32:28,142 DEBUG [StoreOpener-9cc76f2c81b25eccf90da7ca97d0721e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:32:28,143 INFO [StoreOpener-9cc76f2c81b25eccf90da7ca97d0721e-1 {}] regionserver.HStore(327): Store=9cc76f2c81b25eccf90da7ca97d0721e/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T14:32:28,143 INFO [StoreOpener-9cc76f2c81b25eccf90da7ca97d0721e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 9cc76f2c81b25eccf90da7ca97d0721e 2024-11-16T14:32:28,145 INFO [StoreOpener-9cc76f2c81b25eccf90da7ca97d0721e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9cc76f2c81b25eccf90da7ca97d0721e columnFamilyName cf2 2024-11-16T14:32:28,145 DEBUG [StoreOpener-9cc76f2c81b25eccf90da7ca97d0721e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:32:28,145 INFO [StoreOpener-9cc76f2c81b25eccf90da7ca97d0721e-1 {}] regionserver.HStore(327): Store=9cc76f2c81b25eccf90da7ca97d0721e/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T14:32:28,145 DEBUG [RS_OPEN_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 9cc76f2c81b25eccf90da7ca97d0721e 2024-11-16T14:32:28,146 DEBUG [RS_OPEN_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43457/user/jenkins/test-data/0d13f5ec-97ce-91d4-7824-c5174956db56/data/default/testReplayEditsAfterRegionMovedWithMultiCF/9cc76f2c81b25eccf90da7ca97d0721e 2024-11-16T14:32:28,147 DEBUG [RS_OPEN_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43457/user/jenkins/test-data/0d13f5ec-97ce-91d4-7824-c5174956db56/data/default/testReplayEditsAfterRegionMovedWithMultiCF/9cc76f2c81b25eccf90da7ca97d0721e 2024-11-16T14:32:28,148 DEBUG [RS_OPEN_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 9cc76f2c81b25eccf90da7ca97d0721e 2024-11-16T14:32:28,148 DEBUG [RS_OPEN_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 9cc76f2c81b25eccf90da7ca97d0721e 2024-11-16T14:32:28,149 DEBUG [RS_OPEN_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-11-16T14:32:28,151 DEBUG [RS_OPEN_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 9cc76f2c81b25eccf90da7ca97d0721e 2024-11-16T14:32:28,158 DEBUG [RS_OPEN_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43457/user/jenkins/test-data/0d13f5ec-97ce-91d4-7824-c5174956db56/data/default/testReplayEditsAfterRegionMovedWithMultiCF/9cc76f2c81b25eccf90da7ca97d0721e/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-16T14:32:28,159 INFO [RS_OPEN_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 9cc76f2c81b25eccf90da7ca97d0721e; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68011561, jitterRate=0.013451233506202698}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-11-16T14:32:28,159 DEBUG [RS_OPEN_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 9cc76f2c81b25eccf90da7ca97d0721e 2024-11-16T14:32:28,160 DEBUG [RS_OPEN_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 9cc76f2c81b25eccf90da7ca97d0721e: Running coprocessor pre-open hook at 1731767548138Writing region info on filesystem at 1731767548138Initializing all the Stores at 1731767548139 (+1 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731767548139Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731767548140 (+1 ms)Cleaning up temporary data from old regions at 1731767548148 (+8 ms)Running coprocessor post-open hooks at 1731767548159 (+11 ms)Region opened successfully at 1731767548160 (+1 ms) 2024-11-16T14:32:28,161 INFO [RS_OPEN_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1731767547738.9cc76f2c81b25eccf90da7ca97d0721e., pid=6, masterSystemTime=1731767548128 2024-11-16T14:32:28,164 DEBUG [RS_OPEN_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1731767547738.9cc76f2c81b25eccf90da7ca97d0721e. 2024-11-16T14:32:28,164 INFO [RS_OPEN_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1731767547738.9cc76f2c81b25eccf90da7ca97d0721e. 2024-11-16T14:32:28,165 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=9cc76f2c81b25eccf90da7ca97d0721e, regionState=OPEN, openSeqNum=2, regionLocation=04e93f2b5fad,45119,1731767533018 2024-11-16T14:32:28,168 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9cc76f2c81b25eccf90da7ca97d0721e, server=04e93f2b5fad,45119,1731767533018 because future has completed 2024-11-16T14:32:28,173 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-16T14:32:28,174 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 9cc76f2c81b25eccf90da7ca97d0721e, server=04e93f2b5fad,45119,1731767533018 in 196 msec 2024-11-16T14:32:28,176 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-16T14:32:28,176 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=9cc76f2c81b25eccf90da7ca97d0721e, ASSIGN in 368 msec 2024-11-16T14:32:28,178 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-16T14:32:28,178 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testReplayEditsAfterRegionMovedWithMultiCF","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731767548178"}]},"ts":"1731767548178"} 2024-11-16T14:32:28,180 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testReplayEditsAfterRegionMovedWithMultiCF, state=ENABLED in hbase:meta 2024-11-16T14:32:28,182 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_POST_OPERATION 2024-11-16T14:32:28,185 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF in 440 msec 2024-11-16T14:32:28,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34365 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-16T14:32:28,387 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testReplayEditsAfterRegionMovedWithMultiCF get assigned. Timeout = 60000ms 2024-11-16T14:32:28,387 INFO [RPCClient-NioEventLoopGroup-6-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testReplayEditsAfterRegionMovedWithMultiCF completed 2024-11-16T14:32:28,388 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-16T14:32:28,394 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testReplayEditsAfterRegionMovedWithMultiCF assigned to meta. Checking AM states. 2024-11-16T14:32:28,394 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-16T14:32:28,395 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testReplayEditsAfterRegionMovedWithMultiCF assigned. 2024-11-16T14:32:28,407 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1731767547738.9cc76f2c81b25eccf90da7ca97d0721e., hostname=04e93f2b5fad,45119,1731767533018, seqNum=2] 2024-11-16T14:32:28,408 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T14:32:28,410 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39898, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T14:32:28,425 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34365 {}] master.HMaster(2410): Client=jenkins//172.17.0.2 move hri=9cc76f2c81b25eccf90da7ca97d0721e, source=04e93f2b5fad,45119,1731767533018, destination=04e93f2b5fad,35241,1731767532873, warming up region on 04e93f2b5fad,35241,1731767532873 2024-11-16T14:32:28,427 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34365 {}] master.HMaster(2414): Client=jenkins//172.17.0.2 move hri=9cc76f2c81b25eccf90da7ca97d0721e, source=04e93f2b5fad,45119,1731767533018, destination=04e93f2b5fad,35241,1731767532873, running balancer 2024-11-16T14:32:28,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34365 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=9cc76f2c81b25eccf90da7ca97d0721e, REOPEN/MOVE 2024-11-16T14:32:28,429 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=9cc76f2c81b25eccf90da7ca97d0721e, REOPEN/MOVE 2024-11-16T14:32:28,431 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=9cc76f2c81b25eccf90da7ca97d0721e, regionState=CLOSING, regionLocation=04e93f2b5fad,45119,1731767533018 2024-11-16T14:32:28,432 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35241 {}] regionserver.RSRpcServices(2066): Warmup testReplayEditsAfterRegionMovedWithMultiCF,,1731767547738.9cc76f2c81b25eccf90da7ca97d0721e. 2024-11-16T14:32:28,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35241 {}] regionserver.HRegion(7855): Warmup {ENCODED => 9cc76f2c81b25eccf90da7ca97d0721e, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1731767547738.9cc76f2c81b25eccf90da7ca97d0721e.', STARTKEY => '', ENDKEY => ''} 2024-11-16T14:32:28,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35241 {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1731767547738.9cc76f2c81b25eccf90da7ca97d0721e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T14:32:28,433 INFO [StoreOpener-9cc76f2c81b25eccf90da7ca97d0721e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 9cc76f2c81b25eccf90da7ca97d0721e 2024-11-16T14:32:28,435 INFO [StoreOpener-9cc76f2c81b25eccf90da7ca97d0721e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9cc76f2c81b25eccf90da7ca97d0721e columnFamilyName cf1 2024-11-16T14:32:28,435 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=9cc76f2c81b25eccf90da7ca97d0721e, REOPEN/MOVE because future has completed 2024-11-16T14:32:28,435 DEBUG [StoreOpener-9cc76f2c81b25eccf90da7ca97d0721e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:32:28,435 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-16T14:32:28,436 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure 9cc76f2c81b25eccf90da7ca97d0721e, server=04e93f2b5fad,45119,1731767533018}] 2024-11-16T14:32:28,436 INFO [StoreOpener-9cc76f2c81b25eccf90da7ca97d0721e-1 {}] regionserver.HStore(327): Store=9cc76f2c81b25eccf90da7ca97d0721e/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T14:32:28,436 INFO [StoreOpener-9cc76f2c81b25eccf90da7ca97d0721e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 9cc76f2c81b25eccf90da7ca97d0721e 2024-11-16T14:32:28,437 INFO [StoreOpener-9cc76f2c81b25eccf90da7ca97d0721e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9cc76f2c81b25eccf90da7ca97d0721e columnFamilyName cf2 2024-11-16T14:32:28,437 DEBUG [StoreOpener-9cc76f2c81b25eccf90da7ca97d0721e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:32:28,438 INFO [StoreOpener-9cc76f2c81b25eccf90da7ca97d0721e-1 {}] regionserver.HStore(327): Store=9cc76f2c81b25eccf90da7ca97d0721e/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T14:32:28,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35241 {}] regionserver.HRegion(1722): Closing 9cc76f2c81b25eccf90da7ca97d0721e, disabling compactions & flushes 2024-11-16T14:32:28,439 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35241 {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1731767547738.9cc76f2c81b25eccf90da7ca97d0721e. 2024-11-16T14:32:28,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35241 {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731767547738.9cc76f2c81b25eccf90da7ca97d0721e. 2024-11-16T14:32:28,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35241 {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731767547738.9cc76f2c81b25eccf90da7ca97d0721e. after waiting 0 ms 2024-11-16T14:32:28,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35241 {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1731767547738.9cc76f2c81b25eccf90da7ca97d0721e. 2024-11-16T14:32:28,439 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35241 {}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1731767547738.9cc76f2c81b25eccf90da7ca97d0721e. 2024-11-16T14:32:28,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35241 {}] regionserver.HRegion(1676): Region close journal for 9cc76f2c81b25eccf90da7ca97d0721e: Waiting for close lock at 1731767548438Disabling compacts and flushes for region at 1731767548438Disabling writes for close at 1731767548439 (+1 ms)Writing region close event to WAL at 1731767548439Closed at 1731767548439 2024-11-16T14:32:28,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34365 {}] procedure.ProcedureSyncWait(219): waitFor pid=7 2024-11-16T14:32:28,596 INFO [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(122): Close 9cc76f2c81b25eccf90da7ca97d0721e 2024-11-16T14:32:28,596 DEBUG [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-16T14:32:28,597 DEBUG [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1722): Closing 9cc76f2c81b25eccf90da7ca97d0721e, disabling compactions & flushes 2024-11-16T14:32:28,597 INFO [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1731767547738.9cc76f2c81b25eccf90da7ca97d0721e. 2024-11-16T14:32:28,597 DEBUG [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731767547738.9cc76f2c81b25eccf90da7ca97d0721e. 2024-11-16T14:32:28,597 DEBUG [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731767547738.9cc76f2c81b25eccf90da7ca97d0721e. after waiting 0 ms 2024-11-16T14:32:28,597 DEBUG [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1731767547738.9cc76f2c81b25eccf90da7ca97d0721e. 2024-11-16T14:32:28,597 INFO [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(2902): Flushing 9cc76f2c81b25eccf90da7ca97d0721e 2/2 column families, dataSize=31 B heapSize=616 B 2024-11-16T14:32:28,614 DEBUG [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43457/user/jenkins/test-data/0d13f5ec-97ce-91d4-7824-c5174956db56/data/default/testReplayEditsAfterRegionMovedWithMultiCF/9cc76f2c81b25eccf90da7ca97d0721e/.tmp/cf1/43a45a1a627145bb9ae8a3aa33bc73e2 is 35, key is r1/cf1:q/1731767548411/Put/seqid=0 2024-11-16T14:32:28,620 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43279 is added to blk_1073741887_1065 (size=4783) 2024-11-16T14:32:28,621 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33005 is added to blk_1073741887_1065 (size=4783) 2024-11-16T14:32:28,621 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36193 is added to blk_1073741887_1065 (size=4783) 2024-11-16T14:32:28,621 INFO [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31 B at sequenceid=5 (bloomFilter=false), to=hdfs://localhost:43457/user/jenkins/test-data/0d13f5ec-97ce-91d4-7824-c5174956db56/data/default/testReplayEditsAfterRegionMovedWithMultiCF/9cc76f2c81b25eccf90da7ca97d0721e/.tmp/cf1/43a45a1a627145bb9ae8a3aa33bc73e2 2024-11-16T14:32:28,629 DEBUG [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43457/user/jenkins/test-data/0d13f5ec-97ce-91d4-7824-c5174956db56/data/default/testReplayEditsAfterRegionMovedWithMultiCF/9cc76f2c81b25eccf90da7ca97d0721e/.tmp/cf1/43a45a1a627145bb9ae8a3aa33bc73e2 as hdfs://localhost:43457/user/jenkins/test-data/0d13f5ec-97ce-91d4-7824-c5174956db56/data/default/testReplayEditsAfterRegionMovedWithMultiCF/9cc76f2c81b25eccf90da7ca97d0721e/cf1/43a45a1a627145bb9ae8a3aa33bc73e2 2024-11-16T14:32:28,636 INFO [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43457/user/jenkins/test-data/0d13f5ec-97ce-91d4-7824-c5174956db56/data/default/testReplayEditsAfterRegionMovedWithMultiCF/9cc76f2c81b25eccf90da7ca97d0721e/cf1/43a45a1a627145bb9ae8a3aa33bc73e2, entries=1, sequenceid=5, filesize=4.7 K 2024-11-16T14:32:28,637 INFO [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~31 B/31, heapSize ~344 B/344, currentSize=0 B/0 for 9cc76f2c81b25eccf90da7ca97d0721e in 40ms, sequenceid=5, compaction requested=false 2024-11-16T14:32:28,638 DEBUG [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testReplayEditsAfterRegionMovedWithMultiCF' 2024-11-16T14:32:28,643 DEBUG [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43457/user/jenkins/test-data/0d13f5ec-97ce-91d4-7824-c5174956db56/data/default/testReplayEditsAfterRegionMovedWithMultiCF/9cc76f2c81b25eccf90da7ca97d0721e/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-16T14:32:28,645 INFO [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1731767547738.9cc76f2c81b25eccf90da7ca97d0721e. 2024-11-16T14:32:28,645 DEBUG [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1676): Region close journal for 9cc76f2c81b25eccf90da7ca97d0721e: Waiting for close lock at 1731767548597Running coprocessor pre-close hooks at 1731767548597Disabling compacts and flushes for region at 1731767548597Disabling writes for close at 1731767548597Obtaining lock to block concurrent updates at 1731767548597Preparing flush snapshotting stores in 9cc76f2c81b25eccf90da7ca97d0721e at 1731767548597Finished memstore snapshotting testReplayEditsAfterRegionMovedWithMultiCF,,1731767547738.9cc76f2c81b25eccf90da7ca97d0721e., syncing WAL and waiting on mvcc, flushsize=dataSize=31, getHeapSize=584, getOffHeapSize=0, getCellsCount=1 at 1731767548597Flushing stores of testReplayEditsAfterRegionMovedWithMultiCF,,1731767547738.9cc76f2c81b25eccf90da7ca97d0721e. at 1731767548598 (+1 ms)Flushing 9cc76f2c81b25eccf90da7ca97d0721e/cf1: creating writer at 1731767548598Flushing 9cc76f2c81b25eccf90da7ca97d0721e/cf1: appending metadata at 1731767548613 (+15 ms)Flushing 9cc76f2c81b25eccf90da7ca97d0721e/cf1: closing flushed file at 1731767548613Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@455cd14e: reopening flushed file at 1731767548628 (+15 ms)Finished flush of dataSize ~31 B/31, heapSize ~344 B/344, currentSize=0 B/0 for 9cc76f2c81b25eccf90da7ca97d0721e in 40ms, sequenceid=5, compaction requested=false at 1731767548638 (+10 ms)Writing region close event to WAL at 1731767548639 (+1 ms)Running coprocessor post-close hooks at 1731767548643 (+4 ms)Closed at 1731767548645 (+2 ms) 2024-11-16T14:32:28,646 INFO [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionServer(3302): Adding 9cc76f2c81b25eccf90da7ca97d0721e move to 04e93f2b5fad,35241,1731767532873 record at close sequenceid=5 2024-11-16T14:32:28,649 INFO [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(157): Closed 9cc76f2c81b25eccf90da7ca97d0721e 2024-11-16T14:32:28,650 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=9cc76f2c81b25eccf90da7ca97d0721e, regionState=CLOSED 2024-11-16T14:32:28,653 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure 9cc76f2c81b25eccf90da7ca97d0721e, server=04e93f2b5fad,45119,1731767533018 because future has completed 2024-11-16T14:32:28,659 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-16T14:32:28,659 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; CloseRegionProcedure 9cc76f2c81b25eccf90da7ca97d0721e, server=04e93f2b5fad,45119,1731767533018 in 219 msec 2024-11-16T14:32:28,660 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=9cc76f2c81b25eccf90da7ca97d0721e, REOPEN/MOVE; state=CLOSED, location=04e93f2b5fad,35241,1731767532873; forceNewPlan=false, retain=false 2024-11-16T14:32:28,811 INFO [04e93f2b5fad:34365 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-16T14:32:28,811 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=9cc76f2c81b25eccf90da7ca97d0721e, regionState=OPENING, regionLocation=04e93f2b5fad,35241,1731767532873 2024-11-16T14:32:28,814 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=9cc76f2c81b25eccf90da7ca97d0721e, REOPEN/MOVE because future has completed 2024-11-16T14:32:28,814 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9cc76f2c81b25eccf90da7ca97d0721e, server=04e93f2b5fad,35241,1731767532873}] 2024-11-16T14:32:28,972 INFO [RS_OPEN_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1731767547738.9cc76f2c81b25eccf90da7ca97d0721e. 2024-11-16T14:32:28,972 DEBUG [RS_OPEN_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => 9cc76f2c81b25eccf90da7ca97d0721e, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1731767547738.9cc76f2c81b25eccf90da7ca97d0721e.', STARTKEY => '', ENDKEY => ''} 2024-11-16T14:32:28,972 DEBUG [RS_OPEN_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF 9cc76f2c81b25eccf90da7ca97d0721e 2024-11-16T14:32:28,972 DEBUG [RS_OPEN_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1731767547738.9cc76f2c81b25eccf90da7ca97d0721e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T14:32:28,972 DEBUG [RS_OPEN_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for 9cc76f2c81b25eccf90da7ca97d0721e 2024-11-16T14:32:28,972 DEBUG [RS_OPEN_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for 9cc76f2c81b25eccf90da7ca97d0721e 2024-11-16T14:32:28,974 INFO [StoreOpener-9cc76f2c81b25eccf90da7ca97d0721e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 9cc76f2c81b25eccf90da7ca97d0721e 2024-11-16T14:32:28,975 INFO [StoreOpener-9cc76f2c81b25eccf90da7ca97d0721e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9cc76f2c81b25eccf90da7ca97d0721e columnFamilyName cf1 2024-11-16T14:32:28,975 DEBUG [StoreOpener-9cc76f2c81b25eccf90da7ca97d0721e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:32:28,982 DEBUG [StoreOpener-9cc76f2c81b25eccf90da7ca97d0721e-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:43457/user/jenkins/test-data/0d13f5ec-97ce-91d4-7824-c5174956db56/data/default/testReplayEditsAfterRegionMovedWithMultiCF/9cc76f2c81b25eccf90da7ca97d0721e/cf1/43a45a1a627145bb9ae8a3aa33bc73e2 2024-11-16T14:32:28,983 INFO [StoreOpener-9cc76f2c81b25eccf90da7ca97d0721e-1 {}] regionserver.HStore(327): Store=9cc76f2c81b25eccf90da7ca97d0721e/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T14:32:28,983 INFO [StoreOpener-9cc76f2c81b25eccf90da7ca97d0721e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 9cc76f2c81b25eccf90da7ca97d0721e 2024-11-16T14:32:28,984 INFO [StoreOpener-9cc76f2c81b25eccf90da7ca97d0721e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9cc76f2c81b25eccf90da7ca97d0721e columnFamilyName cf2 2024-11-16T14:32:28,984 DEBUG [StoreOpener-9cc76f2c81b25eccf90da7ca97d0721e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:32:28,985 INFO [StoreOpener-9cc76f2c81b25eccf90da7ca97d0721e-1 {}] regionserver.HStore(327): Store=9cc76f2c81b25eccf90da7ca97d0721e/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T14:32:28,985 DEBUG [RS_OPEN_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for 9cc76f2c81b25eccf90da7ca97d0721e 2024-11-16T14:32:28,986 DEBUG [RS_OPEN_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43457/user/jenkins/test-data/0d13f5ec-97ce-91d4-7824-c5174956db56/data/default/testReplayEditsAfterRegionMovedWithMultiCF/9cc76f2c81b25eccf90da7ca97d0721e 2024-11-16T14:32:28,987 DEBUG [RS_OPEN_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43457/user/jenkins/test-data/0d13f5ec-97ce-91d4-7824-c5174956db56/data/default/testReplayEditsAfterRegionMovedWithMultiCF/9cc76f2c81b25eccf90da7ca97d0721e 2024-11-16T14:32:28,988 DEBUG [RS_OPEN_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for 9cc76f2c81b25eccf90da7ca97d0721e 2024-11-16T14:32:28,988 DEBUG [RS_OPEN_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for 9cc76f2c81b25eccf90da7ca97d0721e 2024-11-16T14:32:28,989 DEBUG [RS_OPEN_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-11-16T14:32:28,993 DEBUG [RS_OPEN_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for 9cc76f2c81b25eccf90da7ca97d0721e 2024-11-16T14:32:28,995 INFO [RS_OPEN_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1114): Opened 9cc76f2c81b25eccf90da7ca97d0721e; next sequenceid=9; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71007770, jitterRate=0.05809822678565979}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-11-16T14:32:28,995 DEBUG [RS_OPEN_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 9cc76f2c81b25eccf90da7ca97d0721e 2024-11-16T14:32:28,996 DEBUG [RS_OPEN_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for 9cc76f2c81b25eccf90da7ca97d0721e: Running coprocessor pre-open hook at 1731767548972Writing region info on filesystem at 1731767548972Initializing all the Stores at 1731767548974 (+2 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731767548974Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731767548974Cleaning up temporary data from old regions at 1731767548988 (+14 ms)Running coprocessor post-open hooks at 1731767548995 (+7 ms)Region opened successfully at 1731767548996 (+1 ms) 2024-11-16T14:32:28,998 INFO [RS_OPEN_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1731767547738.9cc76f2c81b25eccf90da7ca97d0721e., pid=9, masterSystemTime=1731767548967 2024-11-16T14:32:29,002 DEBUG [RS_OPEN_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1731767547738.9cc76f2c81b25eccf90da7ca97d0721e. 2024-11-16T14:32:29,002 INFO [RS_OPEN_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1731767547738.9cc76f2c81b25eccf90da7ca97d0721e. 2024-11-16T14:32:29,003 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=9cc76f2c81b25eccf90da7ca97d0721e, regionState=OPEN, openSeqNum=9, regionLocation=04e93f2b5fad,35241,1731767532873 2024-11-16T14:32:29,009 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9cc76f2c81b25eccf90da7ca97d0721e, server=04e93f2b5fad,35241,1731767532873 because future has completed 2024-11-16T14:32:29,014 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=7 2024-11-16T14:32:29,014 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=7, state=SUCCESS, hasLock=false; OpenRegionProcedure 9cc76f2c81b25eccf90da7ca97d0721e, server=04e93f2b5fad,35241,1731767532873 in 197 msec 2024-11-16T14:32:29,016 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=9cc76f2c81b25eccf90da7ca97d0721e, REOPEN/MOVE in 587 msec 2024-11-16T14:32:29,034 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-16T14:32:29,036 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39382, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-16T14:32:29,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45119 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 139 connection: 172.17.0.2:39898 deadline: 1731767609039, exception=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=04e93f2b5fad port=35241 startCode=1731767532873. As of locationSeqNum=5. 2024-11-16T14:32:29,060 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1731767547738.9cc76f2c81b25eccf90da7ca97d0721e., hostname=04e93f2b5fad,45119,1731767533018, seqNum=2 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1731767547738.9cc76f2c81b25eccf90da7ca97d0721e., hostname=04e93f2b5fad,45119,1731767533018, seqNum=2, error=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=04e93f2b5fad port=35241 startCode=1731767532873. As of locationSeqNum=5. 2024-11-16T14:32:29,061 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1731767547738.9cc76f2c81b25eccf90da7ca97d0721e., hostname=04e93f2b5fad,45119,1731767533018, seqNum=2 is org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=04e93f2b5fad port=35241 startCode=1731767532873. As of locationSeqNum=5. 2024-11-16T14:32:29,061 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(84): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1731767547738.9cc76f2c81b25eccf90da7ca97d0721e., hostname=04e93f2b5fad,45119,1731767533018, seqNum=2 with the new location region=testReplayEditsAfterRegionMovedWithMultiCF,,1731767547738.9cc76f2c81b25eccf90da7ca97d0721e., hostname=04e93f2b5fad,35241,1731767532873, seqNum=5 constructed by org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=04e93f2b5fad port=35241 startCode=1731767532873. As of locationSeqNum=5. 2024-11-16T14:32:29,177 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 9cc76f2c81b25eccf90da7ca97d0721e 2/2 column families, dataSize=50 B heapSize=720 B 2024-11-16T14:32:29,195 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43457/user/jenkins/test-data/0d13f5ec-97ce-91d4-7824-c5174956db56/data/default/testReplayEditsAfterRegionMovedWithMultiCF/9cc76f2c81b25eccf90da7ca97d0721e/.tmp/cf1/e166e8ed02994499aeda3b9f39364f85 is 29, key is r1/cf1:/1731767549167/DeleteFamily/seqid=0 2024-11-16T14:32:29,201 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36193 is added to blk_1073741888_1066 (size=4906) 2024-11-16T14:32:29,201 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33005 is added to blk_1073741888_1066 (size=4906) 2024-11-16T14:32:29,201 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43279 is added to blk_1073741888_1066 (size=4906) 2024-11-16T14:32:29,202 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=25 B at sequenceid=12 (bloomFilter=false), to=hdfs://localhost:43457/user/jenkins/test-data/0d13f5ec-97ce-91d4-7824-c5174956db56/data/default/testReplayEditsAfterRegionMovedWithMultiCF/9cc76f2c81b25eccf90da7ca97d0721e/.tmp/cf1/e166e8ed02994499aeda3b9f39364f85 2024-11-16T14:32:29,209 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for e166e8ed02994499aeda3b9f39364f85 2024-11-16T14:32:29,230 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43457/user/jenkins/test-data/0d13f5ec-97ce-91d4-7824-c5174956db56/data/default/testReplayEditsAfterRegionMovedWithMultiCF/9cc76f2c81b25eccf90da7ca97d0721e/.tmp/cf2/1876ea78ccd34488ac732a8a451da0de is 29, key is r1/cf2:/1731767549167/DeleteFamily/seqid=0 2024-11-16T14:32:29,236 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36193 is added to blk_1073741889_1067 (size=4906) 2024-11-16T14:32:29,237 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43279 is added to blk_1073741889_1067 (size=4906) 2024-11-16T14:32:29,237 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33005 is added to blk_1073741889_1067 (size=4906) 2024-11-16T14:32:29,238 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=25 B at sequenceid=12 (bloomFilter=false), to=hdfs://localhost:43457/user/jenkins/test-data/0d13f5ec-97ce-91d4-7824-c5174956db56/data/default/testReplayEditsAfterRegionMovedWithMultiCF/9cc76f2c81b25eccf90da7ca97d0721e/.tmp/cf2/1876ea78ccd34488ac732a8a451da0de 2024-11-16T14:32:29,244 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 1876ea78ccd34488ac732a8a451da0de 2024-11-16T14:32:29,246 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43457/user/jenkins/test-data/0d13f5ec-97ce-91d4-7824-c5174956db56/data/default/testReplayEditsAfterRegionMovedWithMultiCF/9cc76f2c81b25eccf90da7ca97d0721e/.tmp/cf1/e166e8ed02994499aeda3b9f39364f85 as hdfs://localhost:43457/user/jenkins/test-data/0d13f5ec-97ce-91d4-7824-c5174956db56/data/default/testReplayEditsAfterRegionMovedWithMultiCF/9cc76f2c81b25eccf90da7ca97d0721e/cf1/e166e8ed02994499aeda3b9f39364f85 2024-11-16T14:32:29,253 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for e166e8ed02994499aeda3b9f39364f85 2024-11-16T14:32:29,253 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43457/user/jenkins/test-data/0d13f5ec-97ce-91d4-7824-c5174956db56/data/default/testReplayEditsAfterRegionMovedWithMultiCF/9cc76f2c81b25eccf90da7ca97d0721e/cf1/e166e8ed02994499aeda3b9f39364f85, entries=1, sequenceid=12, filesize=4.8 K 2024-11-16T14:32:29,255 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43457/user/jenkins/test-data/0d13f5ec-97ce-91d4-7824-c5174956db56/data/default/testReplayEditsAfterRegionMovedWithMultiCF/9cc76f2c81b25eccf90da7ca97d0721e/.tmp/cf2/1876ea78ccd34488ac732a8a451da0de as hdfs://localhost:43457/user/jenkins/test-data/0d13f5ec-97ce-91d4-7824-c5174956db56/data/default/testReplayEditsAfterRegionMovedWithMultiCF/9cc76f2c81b25eccf90da7ca97d0721e/cf2/1876ea78ccd34488ac732a8a451da0de 2024-11-16T14:32:29,263 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 1876ea78ccd34488ac732a8a451da0de 2024-11-16T14:32:29,263 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43457/user/jenkins/test-data/0d13f5ec-97ce-91d4-7824-c5174956db56/data/default/testReplayEditsAfterRegionMovedWithMultiCF/9cc76f2c81b25eccf90da7ca97d0721e/cf2/1876ea78ccd34488ac732a8a451da0de, entries=1, sequenceid=12, filesize=4.8 K 2024-11-16T14:32:29,264 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~50 B/50, heapSize ~688 B/688, currentSize=0 B/0 for 9cc76f2c81b25eccf90da7ca97d0721e in 87ms, sequenceid=12, compaction requested=false 2024-11-16T14:32:29,265 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 9cc76f2c81b25eccf90da7ca97d0721e: 2024-11-16T14:32:29,268 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-16T14:32:29,270 DEBUG [Time-limited test {}] regionserver.HStore(1541): 9cc76f2c81b25eccf90da7ca97d0721e/cf1 is initiating major compaction (all files) 2024-11-16T14:32:29,270 INFO [Time-limited test {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-16T14:32:29,270 INFO [Time-limited test {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-16T14:32:29,271 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of 9cc76f2c81b25eccf90da7ca97d0721e/cf1 in testReplayEditsAfterRegionMovedWithMultiCF,,1731767547738.9cc76f2c81b25eccf90da7ca97d0721e. 2024-11-16T14:32:29,271 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43457/user/jenkins/test-data/0d13f5ec-97ce-91d4-7824-c5174956db56/data/default/testReplayEditsAfterRegionMovedWithMultiCF/9cc76f2c81b25eccf90da7ca97d0721e/cf1/43a45a1a627145bb9ae8a3aa33bc73e2, hdfs://localhost:43457/user/jenkins/test-data/0d13f5ec-97ce-91d4-7824-c5174956db56/data/default/testReplayEditsAfterRegionMovedWithMultiCF/9cc76f2c81b25eccf90da7ca97d0721e/cf1/e166e8ed02994499aeda3b9f39364f85] into tmpdir=hdfs://localhost:43457/user/jenkins/test-data/0d13f5ec-97ce-91d4-7824-c5174956db56/data/default/testReplayEditsAfterRegionMovedWithMultiCF/9cc76f2c81b25eccf90da7ca97d0721e/.tmp, totalSize=9.5 K 2024-11-16T14:32:29,273 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 43a45a1a627145bb9ae8a3aa33bc73e2, keycount=1, bloomtype=NONE, size=4.7 K, encoding=NONE, compression=NONE, seqNum=5, earliestPutTs=1731767548411 2024-11-16T14:32:29,273 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting e166e8ed02994499aeda3b9f39364f85, keycount=1, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=9223372036854775807 2024-11-16T14:32:29,289 INFO [Time-limited test {}] throttle.PressureAwareThroughputController(145): 9cc76f2c81b25eccf90da7ca97d0721e#cf1#compaction#16 average throughput is NaN MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:32:29,296 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36193 is added to blk_1073741890_1068 (size=4626) 2024-11-16T14:32:29,296 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43279 is added to blk_1073741890_1068 (size=4626) 2024-11-16T14:32:29,296 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33005 is added to blk_1073741890_1068 (size=4626) 2024-11-16T14:32:29,304 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43457/user/jenkins/test-data/0d13f5ec-97ce-91d4-7824-c5174956db56/data/default/testReplayEditsAfterRegionMovedWithMultiCF/9cc76f2c81b25eccf90da7ca97d0721e/.tmp/cf1/455896f41041401abaf9c12db5d9175a as hdfs://localhost:43457/user/jenkins/test-data/0d13f5ec-97ce-91d4-7824-c5174956db56/data/default/testReplayEditsAfterRegionMovedWithMultiCF/9cc76f2c81b25eccf90da7ca97d0721e/cf1/455896f41041401abaf9c12db5d9175a 2024-11-16T14:32:29,319 INFO [Time-limited test {}] regionserver.HStore(1337): Completed major compaction of 2 (all) file(s) in 9cc76f2c81b25eccf90da7ca97d0721e/cf1 of 9cc76f2c81b25eccf90da7ca97d0721e into 455896f41041401abaf9c12db5d9175a(size=4.5 K), total size for store is 4.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:32:29,319 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for 9cc76f2c81b25eccf90da7ca97d0721e: 2024-11-16T14:32:29,320 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 1 store files, 0 compacting, 1 eligible, 16 blocking 2024-11-16T14:32:29,320 DEBUG [Time-limited test {}] regionserver.HStore(1541): 9cc76f2c81b25eccf90da7ca97d0721e/cf2 is initiating major compaction (all files) 2024-11-16T14:32:29,320 INFO [Time-limited test {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-16T14:32:29,320 INFO [Time-limited test {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-16T14:32:29,320 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of 9cc76f2c81b25eccf90da7ca97d0721e/cf2 in testReplayEditsAfterRegionMovedWithMultiCF,,1731767547738.9cc76f2c81b25eccf90da7ca97d0721e. 2024-11-16T14:32:29,320 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43457/user/jenkins/test-data/0d13f5ec-97ce-91d4-7824-c5174956db56/data/default/testReplayEditsAfterRegionMovedWithMultiCF/9cc76f2c81b25eccf90da7ca97d0721e/cf2/1876ea78ccd34488ac732a8a451da0de] into tmpdir=hdfs://localhost:43457/user/jenkins/test-data/0d13f5ec-97ce-91d4-7824-c5174956db56/data/default/testReplayEditsAfterRegionMovedWithMultiCF/9cc76f2c81b25eccf90da7ca97d0721e/.tmp, totalSize=4.8 K 2024-11-16T14:32:29,321 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 1876ea78ccd34488ac732a8a451da0de, keycount=1, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=9223372036854775807 2024-11-16T14:32:29,328 INFO [Time-limited test {}] throttle.PressureAwareThroughputController(145): 9cc76f2c81b25eccf90da7ca97d0721e#cf2#compaction#17 average throughput is NaN MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:32:29,335 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43279 is added to blk_1073741891_1069 (size=4592) 2024-11-16T14:32:29,335 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36193 is added to blk_1073741891_1069 (size=4592) 2024-11-16T14:32:29,336 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33005 is added to blk_1073741891_1069 (size=4592) 2024-11-16T14:32:29,343 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43457/user/jenkins/test-data/0d13f5ec-97ce-91d4-7824-c5174956db56/data/default/testReplayEditsAfterRegionMovedWithMultiCF/9cc76f2c81b25eccf90da7ca97d0721e/.tmp/cf2/1f48dc45b7ab43a29916ce920150edec as hdfs://localhost:43457/user/jenkins/test-data/0d13f5ec-97ce-91d4-7824-c5174956db56/data/default/testReplayEditsAfterRegionMovedWithMultiCF/9cc76f2c81b25eccf90da7ca97d0721e/cf2/1f48dc45b7ab43a29916ce920150edec 2024-11-16T14:32:29,351 INFO [Time-limited test {}] regionserver.HStore(1337): Completed major compaction of 1 (all) file(s) in 9cc76f2c81b25eccf90da7ca97d0721e/cf2 of 9cc76f2c81b25eccf90da7ca97d0721e into 1f48dc45b7ab43a29916ce920150edec(size=4.5 K), total size for store is 4.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:32:29,351 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for 9cc76f2c81b25eccf90da7ca97d0721e: 2024-11-16T14:32:29,356 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34365 {}] master.HMaster(2410): Client=jenkins//172.17.0.2 move hri=9cc76f2c81b25eccf90da7ca97d0721e, source=04e93f2b5fad,35241,1731767532873, destination=04e93f2b5fad,45119,1731767533018, warming up region on 04e93f2b5fad,45119,1731767533018 2024-11-16T14:32:29,356 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34365 {}] master.HMaster(2414): Client=jenkins//172.17.0.2 move hri=9cc76f2c81b25eccf90da7ca97d0721e, source=04e93f2b5fad,35241,1731767532873, destination=04e93f2b5fad,45119,1731767533018, running balancer 2024-11-16T14:32:29,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34365 {}] procedure2.ProcedureExecutor(1139): Stored pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=9cc76f2c81b25eccf90da7ca97d0721e, REOPEN/MOVE 2024-11-16T14:32:29,358 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=9cc76f2c81b25eccf90da7ca97d0721e, REOPEN/MOVE 2024-11-16T14:32:29,360 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=9cc76f2c81b25eccf90da7ca97d0721e, regionState=CLOSING, regionLocation=04e93f2b5fad,35241,1731767532873 2024-11-16T14:32:29,360 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45119 {}] regionserver.RSRpcServices(2066): Warmup testReplayEditsAfterRegionMovedWithMultiCF,,1731767547738.9cc76f2c81b25eccf90da7ca97d0721e. 2024-11-16T14:32:29,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45119 {}] regionserver.HRegion(7855): Warmup {ENCODED => 9cc76f2c81b25eccf90da7ca97d0721e, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1731767547738.9cc76f2c81b25eccf90da7ca97d0721e.', STARTKEY => '', ENDKEY => ''} 2024-11-16T14:32:29,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45119 {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1731767547738.9cc76f2c81b25eccf90da7ca97d0721e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T14:32:29,361 INFO [StoreOpener-9cc76f2c81b25eccf90da7ca97d0721e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 9cc76f2c81b25eccf90da7ca97d0721e 2024-11-16T14:32:29,362 INFO [StoreOpener-9cc76f2c81b25eccf90da7ca97d0721e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9cc76f2c81b25eccf90da7ca97d0721e columnFamilyName cf1 2024-11-16T14:32:29,362 DEBUG [StoreOpener-9cc76f2c81b25eccf90da7ca97d0721e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:32:29,363 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=9cc76f2c81b25eccf90da7ca97d0721e, REOPEN/MOVE because future has completed 2024-11-16T14:32:29,363 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-16T14:32:29,364 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=11, ppid=10, state=RUNNABLE, hasLock=false; CloseRegionProcedure 9cc76f2c81b25eccf90da7ca97d0721e, server=04e93f2b5fad,35241,1731767532873}] 2024-11-16T14:32:29,371 DEBUG [StoreOpener-9cc76f2c81b25eccf90da7ca97d0721e-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:43457/user/jenkins/test-data/0d13f5ec-97ce-91d4-7824-c5174956db56/data/default/testReplayEditsAfterRegionMovedWithMultiCF/9cc76f2c81b25eccf90da7ca97d0721e/cf1/43a45a1a627145bb9ae8a3aa33bc73e2 2024-11-16T14:32:29,376 DEBUG [StoreOpener-9cc76f2c81b25eccf90da7ca97d0721e-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:43457/user/jenkins/test-data/0d13f5ec-97ce-91d4-7824-c5174956db56/data/default/testReplayEditsAfterRegionMovedWithMultiCF/9cc76f2c81b25eccf90da7ca97d0721e/cf1/455896f41041401abaf9c12db5d9175a 2024-11-16T14:32:29,382 INFO [StoreFileOpener-9cc76f2c81b25eccf90da7ca97d0721e-cf1-1 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for e166e8ed02994499aeda3b9f39364f85 2024-11-16T14:32:29,383 DEBUG [StoreOpener-9cc76f2c81b25eccf90da7ca97d0721e-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:43457/user/jenkins/test-data/0d13f5ec-97ce-91d4-7824-c5174956db56/data/default/testReplayEditsAfterRegionMovedWithMultiCF/9cc76f2c81b25eccf90da7ca97d0721e/cf1/e166e8ed02994499aeda3b9f39364f85 2024-11-16T14:32:29,383 INFO [StoreOpener-9cc76f2c81b25eccf90da7ca97d0721e-1 {}] regionserver.HStore(327): Store=9cc76f2c81b25eccf90da7ca97d0721e/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T14:32:29,383 INFO [StoreOpener-9cc76f2c81b25eccf90da7ca97d0721e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 9cc76f2c81b25eccf90da7ca97d0721e 2024-11-16T14:32:29,384 INFO [StoreOpener-9cc76f2c81b25eccf90da7ca97d0721e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9cc76f2c81b25eccf90da7ca97d0721e columnFamilyName cf2 2024-11-16T14:32:29,384 DEBUG [StoreOpener-9cc76f2c81b25eccf90da7ca97d0721e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:32:29,390 INFO [StoreFileOpener-9cc76f2c81b25eccf90da7ca97d0721e-cf2-1 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 1876ea78ccd34488ac732a8a451da0de 2024-11-16T14:32:29,391 DEBUG [StoreOpener-9cc76f2c81b25eccf90da7ca97d0721e-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:43457/user/jenkins/test-data/0d13f5ec-97ce-91d4-7824-c5174956db56/data/default/testReplayEditsAfterRegionMovedWithMultiCF/9cc76f2c81b25eccf90da7ca97d0721e/cf2/1876ea78ccd34488ac732a8a451da0de 2024-11-16T14:32:29,396 DEBUG [StoreOpener-9cc76f2c81b25eccf90da7ca97d0721e-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:43457/user/jenkins/test-data/0d13f5ec-97ce-91d4-7824-c5174956db56/data/default/testReplayEditsAfterRegionMovedWithMultiCF/9cc76f2c81b25eccf90da7ca97d0721e/cf2/1f48dc45b7ab43a29916ce920150edec 2024-11-16T14:32:29,397 INFO [StoreOpener-9cc76f2c81b25eccf90da7ca97d0721e-1 {}] regionserver.HStore(327): Store=9cc76f2c81b25eccf90da7ca97d0721e/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T14:32:29,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45119 {}] regionserver.HRegion(1722): Closing 9cc76f2c81b25eccf90da7ca97d0721e, disabling compactions & flushes 2024-11-16T14:32:29,397 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45119 {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1731767547738.9cc76f2c81b25eccf90da7ca97d0721e. 2024-11-16T14:32:29,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45119 {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731767547738.9cc76f2c81b25eccf90da7ca97d0721e. 2024-11-16T14:32:29,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45119 {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731767547738.9cc76f2c81b25eccf90da7ca97d0721e. after waiting 0 ms 2024-11-16T14:32:29,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45119 {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1731767547738.9cc76f2c81b25eccf90da7ca97d0721e. 2024-11-16T14:32:29,398 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45119 {}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1731767547738.9cc76f2c81b25eccf90da7ca97d0721e. 2024-11-16T14:32:29,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45119 {}] regionserver.HRegion(1676): Region close journal for 9cc76f2c81b25eccf90da7ca97d0721e: Waiting for close lock at 1731767549397Disabling compacts and flushes for region at 1731767549397Disabling writes for close at 1731767549397Writing region close event to WAL at 1731767549398 (+1 ms)Closed at 1731767549398 2024-11-16T14:32:29,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34365 {}] procedure.ProcedureSyncWait(219): waitFor pid=10 2024-11-16T14:32:29,518 INFO [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] handler.UnassignRegionHandler(122): Close 9cc76f2c81b25eccf90da7ca97d0721e 2024-11-16T14:32:29,518 DEBUG [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-16T14:32:29,518 DEBUG [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1722): Closing 9cc76f2c81b25eccf90da7ca97d0721e, disabling compactions & flushes 2024-11-16T14:32:29,518 INFO [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1731767547738.9cc76f2c81b25eccf90da7ca97d0721e. 2024-11-16T14:32:29,518 DEBUG [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731767547738.9cc76f2c81b25eccf90da7ca97d0721e. 2024-11-16T14:32:29,519 DEBUG [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731767547738.9cc76f2c81b25eccf90da7ca97d0721e. after waiting 0 ms 2024-11-16T14:32:29,519 DEBUG [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1731767547738.9cc76f2c81b25eccf90da7ca97d0721e. 2024-11-16T14:32:29,519 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1731767547738.9cc76f2c81b25eccf90da7ca97d0721e.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43457/user/jenkins/test-data/0d13f5ec-97ce-91d4-7824-c5174956db56/data/default/testReplayEditsAfterRegionMovedWithMultiCF/9cc76f2c81b25eccf90da7ca97d0721e/cf1/43a45a1a627145bb9ae8a3aa33bc73e2, hdfs://localhost:43457/user/jenkins/test-data/0d13f5ec-97ce-91d4-7824-c5174956db56/data/default/testReplayEditsAfterRegionMovedWithMultiCF/9cc76f2c81b25eccf90da7ca97d0721e/cf1/e166e8ed02994499aeda3b9f39364f85] to archive 2024-11-16T14:32:29,522 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1731767547738.9cc76f2c81b25eccf90da7ca97d0721e.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-16T14:32:29,525 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1731767547738.9cc76f2c81b25eccf90da7ca97d0721e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43457/user/jenkins/test-data/0d13f5ec-97ce-91d4-7824-c5174956db56/data/default/testReplayEditsAfterRegionMovedWithMultiCF/9cc76f2c81b25eccf90da7ca97d0721e/cf1/43a45a1a627145bb9ae8a3aa33bc73e2 to hdfs://localhost:43457/user/jenkins/test-data/0d13f5ec-97ce-91d4-7824-c5174956db56/archive/data/default/testReplayEditsAfterRegionMovedWithMultiCF/9cc76f2c81b25eccf90da7ca97d0721e/cf1/43a45a1a627145bb9ae8a3aa33bc73e2 2024-11-16T14:32:29,527 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1731767547738.9cc76f2c81b25eccf90da7ca97d0721e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43457/user/jenkins/test-data/0d13f5ec-97ce-91d4-7824-c5174956db56/data/default/testReplayEditsAfterRegionMovedWithMultiCF/9cc76f2c81b25eccf90da7ca97d0721e/cf1/e166e8ed02994499aeda3b9f39364f85 to hdfs://localhost:43457/user/jenkins/test-data/0d13f5ec-97ce-91d4-7824-c5174956db56/archive/data/default/testReplayEditsAfterRegionMovedWithMultiCF/9cc76f2c81b25eccf90da7ca97d0721e/cf1/e166e8ed02994499aeda3b9f39364f85 2024-11-16T14:32:29,538 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1731767547738.9cc76f2c81b25eccf90da7ca97d0721e.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43457/user/jenkins/test-data/0d13f5ec-97ce-91d4-7824-c5174956db56/data/default/testReplayEditsAfterRegionMovedWithMultiCF/9cc76f2c81b25eccf90da7ca97d0721e/cf2/1876ea78ccd34488ac732a8a451da0de] to archive 2024-11-16T14:32:29,539 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1731767547738.9cc76f2c81b25eccf90da7ca97d0721e.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-16T14:32:29,541 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1731767547738.9cc76f2c81b25eccf90da7ca97d0721e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43457/user/jenkins/test-data/0d13f5ec-97ce-91d4-7824-c5174956db56/data/default/testReplayEditsAfterRegionMovedWithMultiCF/9cc76f2c81b25eccf90da7ca97d0721e/cf2/1876ea78ccd34488ac732a8a451da0de to hdfs://localhost:43457/user/jenkins/test-data/0d13f5ec-97ce-91d4-7824-c5174956db56/archive/data/default/testReplayEditsAfterRegionMovedWithMultiCF/9cc76f2c81b25eccf90da7ca97d0721e/cf2/1876ea78ccd34488ac732a8a451da0de 2024-11-16T14:32:29,547 DEBUG [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43457/user/jenkins/test-data/0d13f5ec-97ce-91d4-7824-c5174956db56/data/default/testReplayEditsAfterRegionMovedWithMultiCF/9cc76f2c81b25eccf90da7ca97d0721e/recovered.edits/17.seqid, newMaxSeqId=17, maxSeqId=8 2024-11-16T14:32:29,548 INFO [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1731767547738.9cc76f2c81b25eccf90da7ca97d0721e. 2024-11-16T14:32:29,548 DEBUG [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1676): Region close journal for 9cc76f2c81b25eccf90da7ca97d0721e: Waiting for close lock at 1731767549518Running coprocessor pre-close hooks at 1731767549518Disabling compacts and flushes for region at 1731767549518Disabling writes for close at 1731767549519 (+1 ms)Writing region close event to WAL at 1731767549543 (+24 ms)Running coprocessor post-close hooks at 1731767549548 (+5 ms)Closed at 1731767549548 2024-11-16T14:32:29,548 INFO [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegionServer(3302): Adding 9cc76f2c81b25eccf90da7ca97d0721e move to 04e93f2b5fad,45119,1731767533018 record at close sequenceid=12 2024-11-16T14:32:29,550 INFO [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] handler.UnassignRegionHandler(157): Closed 9cc76f2c81b25eccf90da7ca97d0721e 2024-11-16T14:32:29,551 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=9cc76f2c81b25eccf90da7ca97d0721e, regionState=CLOSED 2024-11-16T14:32:29,553 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=10, state=RUNNABLE, hasLock=false; CloseRegionProcedure 9cc76f2c81b25eccf90da7ca97d0721e, server=04e93f2b5fad,35241,1731767532873 because future has completed 2024-11-16T14:32:29,557 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=11, resume processing ppid=10 2024-11-16T14:32:29,557 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=10, state=SUCCESS, hasLock=false; CloseRegionProcedure 9cc76f2c81b25eccf90da7ca97d0721e, server=04e93f2b5fad,35241,1731767532873 in 191 msec 2024-11-16T14:32:29,558 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=9cc76f2c81b25eccf90da7ca97d0721e, REOPEN/MOVE; state=CLOSED, location=04e93f2b5fad,45119,1731767533018; forceNewPlan=false, retain=false 2024-11-16T14:32:29,709 INFO [04e93f2b5fad:34365 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-16T14:32:29,710 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=9cc76f2c81b25eccf90da7ca97d0721e, regionState=OPENING, regionLocation=04e93f2b5fad,45119,1731767533018 2024-11-16T14:32:29,718 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=9cc76f2c81b25eccf90da7ca97d0721e, REOPEN/MOVE because future has completed 2024-11-16T14:32:29,719 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9cc76f2c81b25eccf90da7ca97d0721e, server=04e93f2b5fad,45119,1731767533018}] 2024-11-16T14:32:29,876 INFO [RS_OPEN_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1731767547738.9cc76f2c81b25eccf90da7ca97d0721e. 2024-11-16T14:32:29,876 DEBUG [RS_OPEN_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7752): Opening region: {ENCODED => 9cc76f2c81b25eccf90da7ca97d0721e, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1731767547738.9cc76f2c81b25eccf90da7ca97d0721e.', STARTKEY => '', ENDKEY => ''} 2024-11-16T14:32:29,877 DEBUG [RS_OPEN_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF 9cc76f2c81b25eccf90da7ca97d0721e 2024-11-16T14:32:29,877 DEBUG [RS_OPEN_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1731767547738.9cc76f2c81b25eccf90da7ca97d0721e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T14:32:29,877 DEBUG [RS_OPEN_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7794): checking encryption for 9cc76f2c81b25eccf90da7ca97d0721e 2024-11-16T14:32:29,877 DEBUG [RS_OPEN_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7797): checking classloading for 9cc76f2c81b25eccf90da7ca97d0721e 2024-11-16T14:32:29,879 INFO [StoreOpener-9cc76f2c81b25eccf90da7ca97d0721e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 9cc76f2c81b25eccf90da7ca97d0721e 2024-11-16T14:32:29,880 INFO [StoreOpener-9cc76f2c81b25eccf90da7ca97d0721e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9cc76f2c81b25eccf90da7ca97d0721e columnFamilyName cf1 2024-11-16T14:32:29,880 DEBUG [StoreOpener-9cc76f2c81b25eccf90da7ca97d0721e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:32:29,887 DEBUG [StoreOpener-9cc76f2c81b25eccf90da7ca97d0721e-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:43457/user/jenkins/test-data/0d13f5ec-97ce-91d4-7824-c5174956db56/data/default/testReplayEditsAfterRegionMovedWithMultiCF/9cc76f2c81b25eccf90da7ca97d0721e/cf1/455896f41041401abaf9c12db5d9175a 2024-11-16T14:32:29,887 INFO [StoreOpener-9cc76f2c81b25eccf90da7ca97d0721e-1 {}] regionserver.HStore(327): Store=9cc76f2c81b25eccf90da7ca97d0721e/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T14:32:29,888 INFO [StoreOpener-9cc76f2c81b25eccf90da7ca97d0721e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 9cc76f2c81b25eccf90da7ca97d0721e 2024-11-16T14:32:29,889 INFO [StoreOpener-9cc76f2c81b25eccf90da7ca97d0721e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9cc76f2c81b25eccf90da7ca97d0721e columnFamilyName cf2 2024-11-16T14:32:29,889 DEBUG [StoreOpener-9cc76f2c81b25eccf90da7ca97d0721e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:32:29,895 DEBUG [StoreOpener-9cc76f2c81b25eccf90da7ca97d0721e-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:43457/user/jenkins/test-data/0d13f5ec-97ce-91d4-7824-c5174956db56/data/default/testReplayEditsAfterRegionMovedWithMultiCF/9cc76f2c81b25eccf90da7ca97d0721e/cf2/1f48dc45b7ab43a29916ce920150edec 2024-11-16T14:32:29,895 INFO [StoreOpener-9cc76f2c81b25eccf90da7ca97d0721e-1 {}] regionserver.HStore(327): Store=9cc76f2c81b25eccf90da7ca97d0721e/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T14:32:29,895 DEBUG [RS_OPEN_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1038): replaying wal for 9cc76f2c81b25eccf90da7ca97d0721e 2024-11-16T14:32:29,896 DEBUG [RS_OPEN_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43457/user/jenkins/test-data/0d13f5ec-97ce-91d4-7824-c5174956db56/data/default/testReplayEditsAfterRegionMovedWithMultiCF/9cc76f2c81b25eccf90da7ca97d0721e 2024-11-16T14:32:29,897 DEBUG [RS_OPEN_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43457/user/jenkins/test-data/0d13f5ec-97ce-91d4-7824-c5174956db56/data/default/testReplayEditsAfterRegionMovedWithMultiCF/9cc76f2c81b25eccf90da7ca97d0721e 2024-11-16T14:32:29,898 DEBUG [RS_OPEN_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1048): stopping wal replay for 9cc76f2c81b25eccf90da7ca97d0721e 2024-11-16T14:32:29,898 DEBUG [RS_OPEN_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1060): Cleaning up temporary data for 9cc76f2c81b25eccf90da7ca97d0721e 2024-11-16T14:32:29,899 DEBUG [RS_OPEN_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-11-16T14:32:29,900 DEBUG [RS_OPEN_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1093): writing seq id for 9cc76f2c81b25eccf90da7ca97d0721e 2024-11-16T14:32:29,901 INFO [RS_OPEN_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1114): Opened 9cc76f2c81b25eccf90da7ca97d0721e; next sequenceid=18; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70758321, jitterRate=0.054381147027015686}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-11-16T14:32:29,901 DEBUG [RS_OPEN_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 9cc76f2c81b25eccf90da7ca97d0721e 2024-11-16T14:32:29,902 DEBUG [RS_OPEN_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1006): Region open journal for 9cc76f2c81b25eccf90da7ca97d0721e: Running coprocessor pre-open hook at 1731767549877Writing region info on filesystem at 1731767549877Initializing all the Stores at 1731767549879 (+2 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731767549879Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731767549879Cleaning up temporary data from old regions at 1731767549898 (+19 ms)Running coprocessor post-open hooks at 1731767549901 (+3 ms)Region opened successfully at 1731767549902 (+1 ms) 2024-11-16T14:32:29,903 INFO [RS_OPEN_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1731767547738.9cc76f2c81b25eccf90da7ca97d0721e., pid=12, masterSystemTime=1731767549872 2024-11-16T14:32:29,905 DEBUG [RS_OPEN_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1731767547738.9cc76f2c81b25eccf90da7ca97d0721e. 2024-11-16T14:32:29,905 INFO [RS_OPEN_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1731767547738.9cc76f2c81b25eccf90da7ca97d0721e. 2024-11-16T14:32:29,906 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=9cc76f2c81b25eccf90da7ca97d0721e, regionState=OPEN, openSeqNum=18, regionLocation=04e93f2b5fad,45119,1731767533018 2024-11-16T14:32:29,908 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=12, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9cc76f2c81b25eccf90da7ca97d0721e, server=04e93f2b5fad,45119,1731767533018 because future has completed 2024-11-16T14:32:29,911 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=10 2024-11-16T14:32:29,911 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=10, state=SUCCESS, hasLock=false; OpenRegionProcedure 9cc76f2c81b25eccf90da7ca97d0721e, server=04e93f2b5fad,45119,1731767533018 in 190 msec 2024-11-16T14:32:29,913 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=9cc76f2c81b25eccf90da7ca97d0721e, REOPEN/MOVE in 555 msec 2024-11-16T14:32:29,960 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-16T14:32:29,962 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39910, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-16T14:32:29,964 ERROR [Time-limited test {}] regionserver.HRegionServer(2442): ***** ABORTING region server 04e93f2b5fad,45119,1731767533018: testing ***** 2024-11-16T14:32:29,964 ERROR [Time-limited test {}] regionserver.HRegionServer(2447): RegionServer abort: loaded coprocessors are: [org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint] 2024-11-16T14:32:29,966 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for java.lang:type=Memory 2024-11-16T14:32:29,967 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=IPC 2024-11-16T14:32:29,970 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Replication 2024-11-16T14:32:29,972 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Server 2024-11-16T14:32:29,979 INFO [Time-limited test {}] regionserver.HRegionServer(2451): Dump of metrics as JSON on abort: { "beans": [ { "name": "java.lang:type=Memory", "modelerType": "sun.management.MemoryImpl", "ObjectPendingFinalizationCount": 0, "HeapMemoryUsage": { "committed": 1048576000, "init": 1048576000, "max": 2306867200, "used": 325209488 }, "NonHeapMemoryUsage": { "committed": 171442176, "init": 7667712, "max": -1, "used": 168717408 }, "Verbose": false, "ObjectName": "java.lang:type=Memory" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=IPC", "modelerType": "RegionServer,sub=IPC", "tag.Context": "regionserver", "tag.Hostname": "04e93f2b5fad", "queueSize": 0, "numCallsInGeneralQueue": 0, "numCallsInReplicationQueue": 0, "numCallsInBulkLoadQueue": 0, "numCallsInPriorityQueue": 0, "numCallsInMetaPriorityQueue": 0, "numOpenConnections": 3, "numActiveHandler": 0, "numActiveGeneralHandler": 0, "numActivePriorityHandler": 0, "numActiveReplicationHandler": 0, "numGeneralCallsDropped": 0, "numLifoModeSwitches": 0, "numCallsInWriteQueue": 0, "numActiveBulkLoadHandler": 0, "numCallsInReadQueue": 0, "numCallsInScanQueue": 0, "numActiveWriteHandler": 0, "numActiveReadHandler": 0, "numActiveScanHandler": 0, "nettyDirectMemoryUsage": 67108864, "nettyTotalPendingOutboundBytes": 0, "nettyMaxPendingOutboundBytes": 0, "receivedBytes": 2071, "exceptions.RegionMovedException": 0, "authenticationSuccesses": 0, "authorizationFailures": 0, "exceptions.requestTooBig": 0, "UnwritableTime_num_ops": 0, "UnwritableTime_min": 0, "UnwritableTime_max": 0, "UnwritableTime_mean": 0, "UnwritableTime_25th_percentile": 0, "UnwritableTime_median": 0, "UnwritableTime_75th_percentile": 0, "UnwritableTime_90th_percentile": 0, "UnwritableTime_95th_percentile": 0, "UnwritableTime_98th_percentile": 0, "UnwritableTime_99th_percentile": 0, "UnwritableTime_99.9th_percentile": 0, "exceptions.OutOfOrderScannerNextException": 0, "exceptions.rpcThrottling": 0, "exceptions.otherExceptions": 0, "ProcessCallTime_num_ops": 8, "ProcessCallTime_min": 0, "ProcessCallTime_max": 22, "ProcessCallTime_mean": 5, "ProcessCallTime_25th_percentile": 5, "ProcessCallTime_median": 11, "ProcessCallTime_75th_percentile": 16, "ProcessCallTime_90th_percentile": 19, "ProcessCallTime_95th_percentile": 20, "ProcessCallTime_98th_percentile": 21, "ProcessCallTime_99th_percentile": 21, "ProcessCallTime_99.9th_percentile": 21, "ProcessCallTime_TimeRangeCount_0-1": 8, "exceptions.callQueueTooBig": 0, "QueueCallTime_num_ops": 8, "QueueCallTime_min": 0, "QueueCallTime_max": 1, "QueueCallTime_mean": 0, "QueueCallTime_25th_percentile": 0, "QueueCallTime_median": 0, "QueueCallTime_75th_percentile": 0, "QueueCallTime_90th_percentile": 0, "QueueCallTime_95th_percentile": 0, "QueueCallTime_98th_percentile": 0, "QueueCallTime_99th_percentile": 0, "QueueCallTime_99.9th_percentile": 0, "QueueCallTime_TimeRangeCount_0-1": 8, "authenticationFailures": 0, "exceptions.multiResponseTooLarge": 0, "exceptions.callDropped": 0, "TotalCallTime_num_ops": 8, "TotalCallTime_min": 0, "TotalCallTime_max": 22, "TotalCallTime_mean": 5, "TotalCallTime_25th_percentile": 5, "TotalCallTime_median": 11, "TotalCallTime_75th_percentile": 16, "TotalCallTime_90th_percentile": 19, "TotalCallTime_95th_percentile": 20, "TotalCallTime_98th_percentile": 21, "TotalCallTime_99th_percentile": 21, "TotalCallTime_99.9th_percentile": 21, "TotalCallTime_TimeRangeCount_0-1": 8, "exceptions.RegionTooBusyException": 0, "exceptions.FailedSanityCheckException": 0, "ResponseSize_num_ops": 8, "ResponseSize_min": 0, "ResponseSize_max": 174, "ResponseSize_mean": 50, "ResponseSize_25th_percentile": 43, "ResponseSize_median": 87, "ResponseSize_75th_percentile": 130, "ResponseSize_90th_percentile": 156, "ResponseSize_95th_percentile": 165, "ResponseSize_98th_percentile": 170, "ResponseSize_99th_percentile": 172, "ResponseSize_99.9th_percentile": 173, "ResponseSize_SizeRangeCount_0-10": 8, "exceptions.UnknownScannerException": 0, "exceptions": 0, "maxOutboundBytesExceeded": 0, "authenticationFallbacks": 0, "exceptions.quotaExceeded": 0, "exceptions.callTimedOut": 0, "exceptions.NotServingRegionException": 0, "authorizationSuccesses": 0, "exceptions.ScannerResetException": 0, "RequestSize_num_ops": 8, "RequestSize_min": 89, "RequestSize_max": 121, "RequestSize_mean": 103, "RequestSize_25th_percentile": 97, "RequestSize_median": 105, "RequestSize_75th_percentile": 113, "RequestSize_90th_percentile": 117, "RequestSize_95th_percentile": 119, "RequestSize_98th_percentile": 120, "RequestSize_99th_percentile": 120, "RequestSize_99.9th_percentile": 120, "RequestSize_SizeRangeCount_0-10": 8, "sentBytes": 348 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Replication", "modelerType": "RegionServer,sub=Replication", "tag.Context": "regionserver", "tag.Hostname": "04e93f2b5fad", "source.shippedHFiles": 0, "Source.ageOfLastShippedOp_num_ops": 0, "Source.ageOfLastShippedOp_min": 0, "Source.ageOfLastShippedOp_max": 0, "Source.ageOfLastShippedOp_mean": 0, "Source.ageOfLastShippedOp_25th_percentile": 0, "Source.ageOfLastShippedOp_median": 0, "Source.ageOfLastShippedOp_75th_percentile": 0, "Source.ageOfLastShippedOp_90th_percentile": 0, "Source.ageOfLastShippedOp_95th_percentile": 0, "Source.ageOfLastShippedOp_98th_percentile": 0, "Source.ageOfLastShippedOp_99th_percentile": 0, "Source.ageOfLastShippedOp_99.9th_percentile": 0, "source.uncleanlyClosedLogs": 0, "source.closedLogsWithUnknownFileLength": 0, "source.walReaderEditsBufferUsage": 0, "source.repeatedLogFileBytes": 0, "source.sizeOfHFileRefsQueue": 0, "source.logReadInBytes": 0, "source.completedRecoverQueues": 0, "source.sizeOfLogQueue": 0, "source.restartedLogReading": 0, "source.failedRecoverQueues": 0, "source.ignoredUncleanlyClosedLogContentsInBytes": 0, "Sink.ageOfLastAppliedOp_num_ops": 0, "Sink.ageOfLastAppliedOp_min": 0, "Sink.ageOfLastAppliedOp_max": 0, "Sink.ageOfLastAppliedOp_mean": 0, "Sink.ageOfLastAppliedOp_25th_percentile": 0, "Sink.ageOfLastAppliedOp_median": 0, "Sink.ageOfLastAppliedOp_75th_percentile": 0, "Sink.ageOfLastAppliedOp_90th_percentile": 0, "Sink.ageOfLastAppliedOp_95th_percentile": 0, "Sink.ageOfLastAppliedOp_98th_percentile": 0, "Sink.ageOfLastAppliedOp_99th_percentile": 0, "Sink.ageOfLastAppliedOp_99.9th_percentile": 0, "source.logEditsRead": 0, "source.numInitializing": 0, "source.shippedOps": 0, "sink.appliedHFiles": 0, "source.logEditsFiltered": 0, "source.shippedBytes": 0, "sink.appliedOps": 0, "source.completedLogs": 0, "source.failedBatches": 0, "sink.failedBatches": 0, "source.shippedBatches": 0, "sink.appliedBatches": 0 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Server", "modelerType": "RegionServer,sub=Server", "tag.zookeeperQuorum": "127.0.0.1:57212", "tag.serverName": "04e93f2b5fad,39921,1731767532970", "tag.clusterId": "6ead4d3f-84d5-440b-9e8d-d5bc0db5a8c1", "tag.Context": "regionserver", "tag.Hostname": "04e93f2b5fad", "regionCount": 0, "storeCount": 0, "hlogFileCount": 1, "hlogFileSize": 0, "storeFileCount": 0, "maxStoreFileCount": 0, "memStoreSize": 0, "memStoreHeapSize": 0, "memStoreOffHeapSize": 0, "storeFileSize": 0, "storeFileSizeGrowthRate": 0.0, "maxStoreFileAge": 0, "minStoreFileAge": 0, "avgStoreFileAge": 0, "numReferenceFiles": 0, "regionServerStartTime": 1731767532970, "averageRegionSize": 0, "storeFileIndexSize": 0, "staticIndexSize": 0, "staticBloomSize": 0, "bloomFilterRequestsCount": 0, "bloomFilterNegativeResultsCount": 0, "bloomFilterEligibleRequestsCount": 0, "mutationsWithoutWALCount": 0, "mutationsWithoutWALSize": 0, "percentFilesLocal": 0.0, "percentFilesLocalSecondaryRegions": 0.0, "totalBytesRead": 10034, "localBytesRead": 10034, "shortCircuitBytesRead": 0, "zeroCopyBytesRead": 0, "splitQueueLength": 0, "compactionQueueLength": 0, "smallCompactionQueueLength": 0, "largeCompactionQueueLength": 0, "flushQueueLength": 0, "blockCacheFreeSize": 922070024, "blockCacheCount": 0, "blockCacheDataBlockCount": 0, "blockCacheSize": 676856, "blockCacheCountHitPercent": 0.0, "blockCacheExpressHitPercent": 0.0, "l1CacheSize": 676856, "l1CacheFreeSize": 922070024, "l1CacheCount": 0, "l1CacheEvictionCount": 0, "l1CacheHitCount": 0, "l1CacheMissCount": 0, "l1CacheHitRatio": 0.0, "l1CacheMissRatio": 0.0, "l2CacheSize": 0, "l2CacheFreeSize": 0, "l2CacheCount": 0, "l2CacheEvictionCount": 0, "l2CacheHitCount": 0, "l2CacheMissCount": 0, "l2CacheHitRatio": 0.0, "l2CacheMissRatio": 0.0, "mobFileCacheCount": 0, "mobFileCacheHitPercent": 0.0, "readRequestRatePerSecond": 0.0, "writeRequestRatePerSecond": 0.0, "ByteBuffAllocatorHeapAllocationBytes": 4782, "ByteBuffAllocatorPoolAllocationBytes": 0, "ByteBuffAllocatorHeapAllocationRatio": 0.0, "ByteBuffAllocatorTotalBufferCount": 186, "ByteBuffAllocatorUsedBufferCount": 0, "activeScanners": 0, "totalRequestCount": 0, "totalRowActionRequestCount": 0, "readRequestCount": 0, "cpRequestCount": 0, "filteredReadRequestCount": 0, "writeRequestCount": 0, "rpcGetRequestCount": 0, "rpcFullScanRequestCount": 0, "rpcScanRequestCount": 0, "rpcMultiRequestCount": 0, "rpcMutateRequestCount": 0, "checkMutateFailedCount": 0, "checkMutatePassedCount": 0, "blockCacheHitCount": 0, "blockCacheHitCountPrimary": 0, "blockCacheHitCachingCount": 0, "blockCacheMissCount": 0, "blockCacheMissCountPrimary": 0, "blockCacheMissCachingCount": 0, "blockCacheEvictionCount": 0, "blockCacheEvictionCountPrimary": 0, "blockCacheFailedInsertionCount": 0, "blockCacheDataMissCount": 0, "blockCacheLeafIndexMissCount": 0, "blockCacheBloomChunkMissCount": 0, "blockCacheMetaMissCount": 0, "blockCacheRootIndexMissCount": 0, "blockCacheIntermediateIndexMissCount": 0, "blockCacheFileInfoMissCount": 0, "blockCacheGeneralBloomMetaMissCount": 0, "blockCacheDeleteFamilyBloomMissCount": 0, "blockCacheTrailerMissCount": 0, "blockCacheDataHitCount": 0, "blockCacheLeafIndexHitCount": 0, "blockCacheBloomChunkHitCount": 0, "blockCacheMetaHitCount": 0, "blockCacheRootIndexHitCount": 0, "blockCacheIntermediateIndexHitCount": 0, "blockCacheFileInfoHitCount": 0, "blockCacheGeneralBloomMetaHitCount": 0, "blockCacheDeleteFamilyBloomHitCount": 0, "blockCacheTrailerHitCount": 0, "updatesBlockedTime": 0, "flushedCellsCount": 0, "compactedCellsCount": 0, "majorCompactedCellsCount": 0, "flushedCellsSize": 0, "compactedCellsSize": 0, "majorCompactedCellsSize": 0, "cellsCountCompactedFromMob": 0, "cellsCountCompactedToMob": 0, "cellsSizeCompactedFromMob": 0, "cellsSizeCompactedToMob": 0, "mobFlushCount": 0, "mobFlushedCellsCount": 0, "mobFlushedCellsSize": 0, "mobScanCellsCount": 0, "mobScanCellsSize": 0, "mobFileCacheAccessCount": 0, "mobFileCacheMissCount": 0, "mobFileCacheEvictedCount": 0, "hedgedReads": 0, "hedgedReadWins": 0, "hedgedReadOpsInCurThread": 0, "blockedRequestCount": 0, "CheckAndMutate_num_ops": 0, "CheckAndMutate_min": 0, "CheckAndMutate_max": 0, "CheckAndMutate_mean": 0, "CheckAndMutate_25th_percentile": 0, "CheckAndMutate_median": 0, "CheckAndMutate_75th_percentile": 0, "CheckAndMutate_90th_percentile": 0, "CheckAndMutate_95th_percentile": 0, "CheckAndMutate_98th_percentile": 0, "CheckAndMutate_99th_percentile": 0, "CheckAndMutate_99.9th_percentile": 0, "MajorCompactionTime_num_ops": 0, "MajorCompactionTime_min": 0, "MajorCompactionTime_max": 0, "MajorCompactionTime_mean": 0, "MajorCompactionTime_25th_percentile": 0, "MajorCompactionTime_median": 0, "MajorCompactionTime_75th_percentile": 0, "MajorCompactionTime_90th_percentile": 0, "MajorCompactionTime_95th_percentile": 0, "MajorCompactionTime_98th_percentile": 0, "MajorCompactionTime_99th_percentile": 0, "MajorCompactionTime_99.9th_percentile": 0, "ScanTime_num_ops": 0, "ScanTime_min": 0, "ScanTime_max": 0, "ScanTime_mean": 0, "ScanTime_25th_percentile": 0, "ScanTime_median": 0, "ScanTime_75th_percentile": 0, "ScanTime_90th_percentile": 0, "ScanTime_95th_percentile": 0, "ScanTime_98th_percentile": 0, "ScanTime_99th_percentile": 0, "ScanTime_99.9th_percentile": 0, "CheckAndMutateBlockBytesScanned_num_ops": 0, "CheckAndMutateBlockBytesScanned_min": 0, "CheckAndMutateBlockBytesScanned_max": 0, "CheckAndMutateBlockBytesScanned_mean": 0, "CheckAndMutateBlockBytesScanned_25th_percentile": 0, "CheckAndMutateBlockBytesScanned_median": 0, "CheckAndMutateBlockBytesScanned_75th_percentile": 0, "CheckAndMutateBlockBytesScanned_90th_percentile": 0, "CheckAndMutateBlockBytesScanned_95th_percentile": 0, "CheckAndMutateBlockBytesScanned_98th_percentile": 0, "CheckAndMutateBlockBytesScanned_99th_percentile": 0, "CheckAndMutateBlockBytesScanned_99.9th_percentile": 0, "Put_num_ops": 0, "Put_min": 0, "Put_max": 0, "Put_mean": 0, "Put_25th_percentile": 0, "Put_median": 0, "Put_75th_percentile": 0, "Put_90th_percentile": 0, "Put_95th_percentile": 0, "Put_98th_percentile": 0, "Put_99th_percentile": 0, "Put_99.9th_percentile": 0, "splitRequestCount": 0, "AppendBlockBytesScanned_num_ops": 0, "AppendBlockBytesScanned_min": 0, "AppendBlockBytesScanned_max": 0, "AppendBlockBytesScanned_mean": 0, "AppendBlockBytesScanned_25th_percentile": 0, "AppendBlockBytesScanned_median": 0, "AppendBlockBytesScanned_75th_percentile": 0, "AppendBlockBytesScanned_90th_percentile": 0, "AppendBlockBytesScanned_95th_percentile": 0, "AppendBlockBytesScanned_98th_percentile": 0, "AppendBlockBytesScanned_99th_percentile": 0, "AppendBlockBytesScanned_99.9th_percentile": 0, "PutBatch_num_ops": 0, "PutBatch_min": 0, "PutBatch_max": 0, "PutBatch_mean": 0, "PutBatch_25th_percentile": 0, "PutBatch_median": 0, "PutBatch_75th_percentile": 0, "PutBatch_90th_percentile": 0, "PutBatch_95th_percentile": 0, "PutBatch_98th_percentile": 0, "PutBatch_99th_percentile": 0, "PutBatch_99.9th_percentile": 0, "IncrementBlockBytesScanned_num_ops": 0, "IncrementBlockBytesScanned_min": 0, "IncrementBlockBytesScanned_max": 0, "IncrementBlockBytesScanned_mean": 0, "IncrementBlockBytesScanned_25th_percentile": 0, "IncrementBlockBytesScanned_median": 0, "IncrementBlockBytesScanned_75th_percentile": 0, "IncrementBlockBytesScanned_90th_percentile": 0, "IncrementBlockBytesScanned_95th_percentile": 0, "IncrementBlockBytesScanned_98th_percentile": 0, "IncrementBlockBytesScanned_99th_percentile": 0, "IncrementBlockBytesScanned_99.9th_percentile": 0, "SplitTime_num_ops": 0, "SplitTime_min": 0, "SplitTime_max": 0, "SplitTime_mean": 0, "SplitTime_25th_percentile": 0, "SplitTime_median": 0, "SplitTime_75th_percentile": 0, "SplitTime_90th_percentile": 0, "SplitTime_95th_percentile": 0, "SplitTime_98th_percentile": 0, "SplitTime_99th_percentile": 0, "SplitTime_99.9th_percentile": 0, "GetBlockBytesScanned_num_ops": 0, "GetBlockBytesScanned_min": 0, "GetBlockBytesScanned_max": 0, "GetBlockBytesScanned_mean": 0, "GetBlockBytesScanned_25th_percentile": 0, "GetBlockBytesScanned_median": 0, "GetBlockBytesScanned_75th_percentile": 0, "GetBlockBytesScanned_90th_percentile": 0, "GetBlockBytesScanned_95th_percentile": 0, "GetBlockBytesScanned_98th_percentile": 0, "GetBlockBytesScanned_99th_percentile": 0, "GetBlockBytesScanned_99.9th_percentile": 0, "majorCompactedInputBytes": 0, "slowAppendCount": 0, "flushedOutputBytes": 0, "Replay_num_ops": 0, "Replay_min": 0, "Replay_max": 0, "Replay_mean": 0, "Replay_25th_percentile": 0, "Replay_median": 0, "Replay_75th_percentile": 0, "Replay_90th_percentile": 0, "Replay_95th_percentile": 0, "Replay_98th_percentile": 0, "Replay_99th_percentile": 0, "Replay_99.9th_percentile": 0, "MajorCompactionInputSize_num_ops": 0, "MajorCompactionInputSize_min": 0, "MajorCompactionInputSize_max": 0, "MajorCompactionInputSize_mean": 0, "MajorCompactionInputSize_25th_percentile": 0, "MajorCompactionInputSize_median": 0, "MajorCompactionInputSize_75th_percentile": 0, "MajorCompactionInputSize_90th_percentile": 0, "MajorCompactionInputSize_95th_percentile": 0, "MajorCompactionInputSize_98th_percentile": 0, "MajorCompactionInputSize_99th_percentile": 0, "MajorCompactionInputSize_99.9th_percentile": 0, "pauseInfoThresholdExceeded": 0, "CheckAndDelete_num_ops": 0, "CheckAndDelete_min": 0, "CheckAndDelete_max": 0, "CheckAndDelete_mean": 0, "CheckAndDelete_25th_percentile": 0, "CheckAndDelete_median": 0, "CheckAndDelete_75th_percentile": 0, "CheckAndDelete_90th_percentile": 0, "CheckAndDelete_95th_percentile": 0, "CheckAndDelete_98th_percentile": 0, "CheckAndDelete_99th_percentile": 0, "CheckAndDelete_99.9th_percentile": 0, "CompactionInputSize_num_ops": 0, "CompactionInputSize_min": 0, "CompactionInputSize_max": 0, "CompactionInputSize_mean": 0, "CompactionInputSize_25th_percentile": 0, "CompactionInputSize_median": 0, "CompactionInputSize_75th_percentile": 0, "CompactionInputSize_90th_percentile": 0, "CompactionInputSize_95th_percentile": 0, "CompactionInputSize_98th_percentile": 0, "CompactionInputSize_99th_percentile": 0, "CompactionInputSize_99.9th_percentile": 0, "flushedMemstoreBytes": 0, "majorCompactedOutputBytes": 0, "slowPutCount": 0, "compactedInputBytes": 0, "FlushOutputSize_num_ops": 0, "FlushOutputSize_min": 0, "FlushOutputSize_max": 0, "FlushOutputSize_mean": 0, "FlushOutputSize_25th_percentile": 0, "FlushOutputSize_median": 0, "FlushOutputSize_75th_percentile": 0, "FlushOutputSize_90th_percentile": 0, "FlushOutputSize_95th_percentile": 0, "FlushOutputSize_98th_percentile": 0, "FlushOutputSize_99th_percentile": 0, "FlushOutputSize_99.9th_percentile": 0, "PauseTimeWithGc_num_ops": 0, "PauseTimeWithGc_min": 0, "PauseTimeWithGc_max": 0, "PauseTimeWithGc_mean": 0, "PauseTimeWithGc_25th_percentile": 0, "PauseTimeWithGc_median": 0, "PauseTimeWithGc_75th_percentile": 0, "PauseTimeWithGc_90th_percentile": 0, "PauseTimeWithGc_95th_percentile": 0, "PauseTimeWithGc_98th_percentile": 0, "PauseTimeWithGc_99th_percentile": 0, "PauseTimeWithGc_99.9th_percentile": 0, "compactedOutputBytes": 0, "pauseWarnThresholdExceeded": 0, "ScanBlockBytesScanned_num_ops": 0, "ScanBlockBytesScanned_min": 0, "ScanBlockBytesScanned_max": 0, "ScanBlockBytesScanned_mean": 0, "ScanBlockBytesScanned_25th_percentile": 0, "ScanBlockBytesScanned_median": 0, "ScanBlockBytesScanned_75th_percentile": 0, "ScanBlockBytesScanned_90th_percentile": 0, "ScanBlockBytesScanned_95th_percentile": 0, "ScanBlockBytesScanned_98th_percentile": 0, "ScanBlockBytesScanned_99th_percentile": 0, "ScanBlockBytesScanned_99.9th_percentile": 0, "Increment_num_ops": 0, "Increment_min": 0, "Increment_max": 0, "Increment_mean": 0, "Increment_25th_percentile": 0, "Increment_median": 0, "Increment_75th_percentile": 0, "Increment_90th_percentile": 0, "Increment_95th_percentile": 0, "Increment_98th_percentile": 0, "Increment_99th_percentile": 0, "Increment_99.9th_percentile": 0, "Delete_num_ops": 0, "Delete_min": 0, "Delete_max": 0, "Delete_mean": 0, "Delete_25th_percentile": 0, "Delete_median": 0, "Delete_75th_percentile": 0, "Delete_90th_percentile": 0, "Delete_95th_percentile": 0, "Delete_98th_percentile": 0, "Delete_99th_percentile": 0, "Delete_99.9th_percentile": 0, "DeleteBatch_num_ops": 0, "DeleteBatch_min": 0, "DeleteBatch_max": 0, "DeleteBatch_mean": 0, "DeleteBatch_25th_percentile": 0, "DeleteBatch_median": 0, "DeleteBatch_75th_percentile": 0, "DeleteBatch_90th_percentile": 0, "DeleteBatch_95th_percentile": 0, "DeleteBatch_98th_percentile": 0, "DeleteBatch_99th_percentile": 0, "DeleteBatch_99.9th_percentile": 0, "blockBytesScannedCount": 0, "FlushMemstoreSize_num_ops": 0, "FlushMemstoreSize_min": 0, "FlushMemstoreSize_max": 0, "FlushMemstoreSize_mean": 0, "FlushMemstoreSize_25th_percentile": 0, "FlushMemstoreSize_median": 0, "FlushMemstoreSize_75th_percentile": 0, "FlushMemstoreSize_90th_percentile": 0, "FlushMemstoreSize_95th_percentile": 0, "FlushMemstoreSize_98th_percentile": 0, "FlushMemstoreSize_99th_percentile": 0, "FlushMemstoreSize_99.9th_percentile": 0, "CompactionInputFileCount_num_ops": 0, "CompactionInputFileCount_min": 0, "CompactionInputFileCount_max": 0, "CompactionInputFileCount_mean": 0, "CompactionInputFileCount_25th_percentile": 0, "CompactionInputFileCount_median": 0, "CompactionInputFileCount_75th_percentile": 0, "CompactionInputFileCount_90th_percentile": 0, "CompactionInputFileCount_95th_percentile": 0, "CompactionInputFileCount_98th_percentile": 0, "CompactionInputFileCount_99th_percentile": 0, "CompactionInputFileCount_99.9th_percentile": 0, "CompactionTime_num_ops": 0, "CompactionTime_min": 0, "CompactionTime_max": 0, "CompactionTime_mean": 0, "CompactionTime_25th_percentile": 0, "CompactionTime_median": 0, "CompactionTime_75th_percentile": 0, "CompactionTime_90th_percentile": 0, "CompactionTime_95th_percentile": 0, "CompactionTime_98th_percentile": 0, "CompactionTime_99th_percentile": 0, "CompactionTime_99.9th_percentile": 0, "Get_num_ops": 0, "Get_min": 0, "Get_max": 0, "Get_mean": 0, "Get_25th_percentile": 0, "Get_median": 0, "Get_75th_percentile": 0, "Get_90th_percentile": 0, "Get_95th_percentile": 0, "Get_98th_percentile": 0, "Get_99th_percentile": 0, "Get_99.9th_percentile": 0, "MajorCompactionInputFileCount_num_ops": 0, "MajorCompactionInputFileCount_min": 0, "MajorCompactionInputFileCount_max": 0, "MajorCompactionInputFileCount_mean": 0, "MajorCompactionInputFileCount_25th_percentile": 0, "MajorCompactionInputFileCount_median": 0, "MajorCompactionInputFileCount_75th_percentile": 0, "MajorCompactionInputFileCount_90th_percentile": 0, "MajorCompactionInputFileCount_95th_percentile": 0, "MajorCompactionInputFileCount_98th_percentile": 0, "MajorCompactionInputFileCount_99th_percentile": 0, "MajorCompactionInputFileCount_99.9th_percentile": 0, "scannerLeaseExpiredCount": 0, "CheckAndPut_num_ops": 0, "CheckAndPut_min": 0, "CheckAndPut_max": 0, "CheckAndPut_mean": 0, "CheckAndPut_25th_percentile": 0, "CheckAndPut_median": 0, "CheckAndPut_75th_percentile": 0, "CheckAndPut_90th_percentile": 0, "CheckAndPut_95th_percentile": 0, "CheckAndPut_98th_percentile": 0, "CheckAndPut_99th_percentile": 0, "CheckAndPut_99.9th_percentile": 0, "MajorCompactionOutputSize_num_ops": 0, "MajorCompactionOutputSize_min": 0, "MajorCompactionOutputSize_max": 0, "MajorCompactionOutputSize_mean": 0, "MajorCompactionOutputSize_25th_percentile": 0, "MajorCompactionOutputSize_median": 0, "MajorCompactionOutputSize_75th_percentile": 0, "MajorCompactionOutputSize_90th_percentile": 0, "MajorCompactionOutputSize_95th_percentile": 0, "MajorCompactionOutputSize_98th_percentile": 0, "MajorCompactionOutputSize_99th_percentile": 0, "MajorCompactionOutputSize_99.9th_percentile": 0, "CompactionOutputFileCount_num_ops": 0, "CompactionOutputFileCount_min": 0, "CompactionOutputFileCount_max": 0, "CompactionOutputFileCount_mean": 0, "CompactionOutputFileCount_25th_percentile": 0, "CompactionOutputFileCount_median": 0, "CompactionOutputFileCount_75th_percentile": 0, "CompactionOutputFileCount_90th_percentile": 0, "CompactionOutputFileCount_95th_percentile": 0, "CompactionOutputFileCount_98th_percentile": 0, "CompactionOutputFileCount_99th_percentile": 0, "CompactionOutputFileCount_99.9th_percentile": 0, "slowDeleteCount": 0, "FlushTime_num_ops": 0, "FlushTime_min": 0, "FlushTime_max": 0, "FlushTime_mean": 0, "FlushTime_25th_percentile": 0, "FlushTime_median": 0, "FlushTime_75th_percentile": 0, "FlushTime_90th_percentile": 0, "FlushTime_95th_percentile": 0, "FlushTime_98th_percentile": 0, "FlushTime_99th_percentile": 0, "FlushTime_99.9th_percentile": 0, "splitSuccessCount": 0, "MajorCompactionOutputFileCount_num_ops": 0, "MajorCompactionOutputFileCount_min": 0, "MajorCompactionOutputFileCount_max": 0, "MajorCompactionOutputFileCount_mean": 0, "MajorCompactionOutputFileCount_25th_percentile": 0, "MajorCompactionOutputFileCount_median": 0, "MajorCompactionOutputFileCount_75th_percentile": 0, "MajorCompactionOutputFileCount_90th_percentile": 0, "MajorCompactionOutputFileCount_95th_percentile": 0, "MajorCompactionOutputFileCount_98th_percentile": 0, "MajorCompactionOutputFileCount_99th_percentile": 0, "MajorCompactionOutputFileCount_99.9th_percentile": 0, "slowGetCount": 0, "ScanSize_num_ops": 0, "ScanSize_min": 0, "ScanSize_max": 0, "ScanSize_mean": 0, "ScanSize_25th_percentile": 0, "ScanSize_median": 0, "ScanSize_75th_percentile": 0, "ScanSize_90th_percentile": 0, "ScanSize_95th_percentile": 0, "ScanSize_98th_percentile": 0, "ScanSize_99th_percentile": 0, "ScanSize_99.9th_percentile": 0, "CompactionOutputSize_num_ops": 0, "CompactionOutputSize_min": 0, "CompactionOutputSize_max": 0, "CompactionOutputSize_mean": 0, "CompactionOutputSize_25th_percentile": 0, "CompactionOutputSize_median": 0, "CompactionOutputSize_75th_percentile": 0, "CompactionOutputSize_90th_percentile": 0, "CompactionOutputSize_95th_percentile": 0, "CompactionOutputSize_98th_percentile": 0, "CompactionOutputSize_99th_percentile": 0, "CompactionOutputSize_99.9th_percentile": 0, "PauseTimeWithoutGc_num_ops": 0, "PauseTimeWithoutGc_min": 0, "PauseTimeWithoutGc_max": 0, "PauseTimeWithoutGc_mean": 0, "PauseTimeWithoutGc_25th_percentile": 0, "PauseTimeWithoutGc_median": 0, "PauseTimeWithoutGc_75th_percentile": 0, "PauseTimeWithoutGc_90th_percentile": 0, "PauseTimeWithoutGc_95th_percentile": 0, "PauseTimeWithoutGc_98th_percentile": 0, "PauseTimeWithoutGc_99th_percentile": 0, "PauseTimeWithoutGc_99.9th_percentile": 0, "slowIncrementCount": 0, "Append_num_ops": 0, "Append_min": 0, "Append_max": 0, "Append_mean": 0, "Append_25th_percentile": 0, "Append_median": 0, "Append_75th_percentile": 0, "Append_90th_percentile": 0, "Append_95th_percentile": 0, "Append_98th_percentile": 0, "Append_99th_percentile": 0, "Append_99.9th_percentile": 0, "Bulkload_count": 0, "Bulkload_mean_rate": 0.0, "Bulkload_1min_rate": 0.0, "Bulkload_5min_rate": 0.0, "Bulkload_15min_rate": 0.0, "Bulkload_num_ops": 0, "Bulkload_min": 0, "Bulkload_max": 0, "Bulkload_mean": 0, "Bulkload_25th_percentile": 0, "Bulkload_median": 0, "Bulkload_75th_percentile": 0, "Bulkload_90th_percentile": 0, "Bulkload_95th_percentile": 0, "Bulkload_98th_percentile": 0, "Bulkload_99th_percentile": 0, "Bulkload_99.9th_percentile": 0 } ] } 2024-11-16T14:32:29,982 WARN [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34365 {}] master.MasterRpcServices(700): 04e93f2b5fad,45119,1731767533018 reported a fatal error: ***** ABORTING region server 04e93f2b5fad,45119,1731767533018: testing ***** 2024-11-16T14:32:29,984 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '04e93f2b5fad,45119,1731767533018' ***** 2024-11-16T14:32:29,984 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: testing 2024-11-16T14:32:29,984 INFO [RS:2;04e93f2b5fad:45119 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-16T14:32:29,985 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-16T14:32:29,985 INFO [RS:2;04e93f2b5fad:45119 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager abruptly. 2024-11-16T14:32:29,985 INFO [RS:2;04e93f2b5fad:45119 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager abruptly. 2024-11-16T14:32:29,985 INFO [RS:2;04e93f2b5fad:45119 {}] regionserver.HRegionServer(3091): Received CLOSE for 9cc76f2c81b25eccf90da7ca97d0721e 2024-11-16T14:32:29,985 INFO [RS:2;04e93f2b5fad:45119 {}] regionserver.HRegionServer(956): aborting server 04e93f2b5fad,45119,1731767533018 2024-11-16T14:32:29,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35241 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Get size: 140 connection: 172.17.0.2:34404 deadline: 1731767609985, exception=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=04e93f2b5fad port=45119 startCode=1731767533018. As of locationSeqNum=12. 2024-11-16T14:32:29,986 INFO [RS:2;04e93f2b5fad:45119 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-16T14:32:29,986 DEBUG [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 9cc76f2c81b25eccf90da7ca97d0721e, disabling compactions & flushes 2024-11-16T14:32:29,986 INFO [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1731767547738.9cc76f2c81b25eccf90da7ca97d0721e. 2024-11-16T14:32:29,986 INFO [RS:2;04e93f2b5fad:45119 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:2;04e93f2b5fad:45119. 2024-11-16T14:32:29,986 DEBUG [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731767547738.9cc76f2c81b25eccf90da7ca97d0721e. 2024-11-16T14:32:29,986 DEBUG [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731767547738.9cc76f2c81b25eccf90da7ca97d0721e. after waiting 0 ms 2024-11-16T14:32:29,986 DEBUG [RS:2;04e93f2b5fad:45119 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-16T14:32:29,986 DEBUG [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1731767547738.9cc76f2c81b25eccf90da7ca97d0721e. 2024-11-16T14:32:29,986 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1731767547738.9cc76f2c81b25eccf90da7ca97d0721e., hostname=04e93f2b5fad,35241,1731767532873, seqNum=5 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1731767547738.9cc76f2c81b25eccf90da7ca97d0721e., hostname=04e93f2b5fad,35241,1731767532873, seqNum=5, error=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=04e93f2b5fad port=45119 startCode=1731767533018. As of locationSeqNum=12. 2024-11-16T14:32:29,986 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1731767547738.9cc76f2c81b25eccf90da7ca97d0721e., hostname=04e93f2b5fad,35241,1731767532873, seqNum=5 is org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=04e93f2b5fad port=45119 startCode=1731767533018. As of locationSeqNum=12. 2024-11-16T14:32:29,986 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(84): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1731767547738.9cc76f2c81b25eccf90da7ca97d0721e., hostname=04e93f2b5fad,35241,1731767532873, seqNum=5 with the new location region=testReplayEditsAfterRegionMovedWithMultiCF,,1731767547738.9cc76f2c81b25eccf90da7ca97d0721e., hostname=04e93f2b5fad,45119,1731767533018, seqNum=12 constructed by org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=04e93f2b5fad port=45119 startCode=1731767533018. As of locationSeqNum=12. 2024-11-16T14:32:29,986 DEBUG [RS:2;04e93f2b5fad:45119 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:32:29,987 INFO [RS:2;04e93f2b5fad:45119 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-16T14:32:29,987 DEBUG [RS:2;04e93f2b5fad:45119 {}] regionserver.HRegionServer(1325): Online Regions={9cc76f2c81b25eccf90da7ca97d0721e=testReplayEditsAfterRegionMovedWithMultiCF,,1731767547738.9cc76f2c81b25eccf90da7ca97d0721e.} 2024-11-16T14:32:29,987 DEBUG [RS:2;04e93f2b5fad:45119 {}] regionserver.HRegionServer(1351): Waiting on 9cc76f2c81b25eccf90da7ca97d0721e 2024-11-16T14:32:29,988 INFO [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1731767547738.9cc76f2c81b25eccf90da7ca97d0721e. 2024-11-16T14:32:29,988 DEBUG [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 9cc76f2c81b25eccf90da7ca97d0721e: Waiting for close lock at 1731767549985Running coprocessor pre-close hooks at 1731767549986 (+1 ms)Disabling compacts and flushes for region at 1731767549986Disabling writes for close at 1731767549986Writing region close event to WAL at 1731767549988 (+2 ms)Running coprocessor post-close hooks at 1731767549988Closed at 1731767549988 2024-11-16T14:32:29,989 DEBUG [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1731767547738.9cc76f2c81b25eccf90da7ca97d0721e. 2024-11-16T14:32:30,069 INFO [regionserver/04e93f2b5fad:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-16T14:32:30,098 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server 04e93f2b5fad,45119,1731767533018 aborting at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processRequest(ServerRpcConnection.java:564) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processOneRpc(ServerRpcConnection.java:364) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyServerRpcConnection.process(NettyServerRpcConnection.java:89) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:56) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:31) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:32:30,099 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1731767547738.9cc76f2c81b25eccf90da7ca97d0721e., hostname=04e93f2b5fad,45119,1731767533018, seqNum=12 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1731767547738.9cc76f2c81b25eccf90da7ca97d0721e., hostname=04e93f2b5fad,45119,1731767533018, seqNum=12, error=org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server 04e93f2b5fad,45119,1731767533018 aborting 2024-11-16T14:32:30,099 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1731767547738.9cc76f2c81b25eccf90da7ca97d0721e., hostname=04e93f2b5fad,45119,1731767533018, seqNum=12 is org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server 04e93f2b5fad,45119,1731767533018 aborting 2024-11-16T14:32:30,099 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testReplayEditsAfterRegionMovedWithMultiCF,,1731767547738.9cc76f2c81b25eccf90da7ca97d0721e., hostname=04e93f2b5fad,45119,1731767533018, seqNum=12 from cache 2024-11-16T14:32:30,188 INFO [RS:2;04e93f2b5fad:45119 {}] regionserver.HRegionServer(976): stopping server 04e93f2b5fad,45119,1731767533018; all regions closed. 2024-11-16T14:32:30,194 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43279 is added to blk_1073741833_1009 (size=1407) 2024-11-16T14:32:30,195 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36193 is added to blk_1073741833_1009 (size=1407) 2024-11-16T14:32:30,195 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33005 is added to blk_1073741833_1009 (size=1407) 2024-11-16T14:32:30,195 DEBUG [RS:2;04e93f2b5fad:45119 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:32:30,195 INFO [RS:2;04e93f2b5fad:45119 {}] regionserver.LeaseManager(133): Closed leases 2024-11-16T14:32:30,196 INFO [RS:2;04e93f2b5fad:45119 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-16T14:32:30,196 INFO [RS:2;04e93f2b5fad:45119 {}] hbase.ChoreService(370): Chore service for: regionserver/04e93f2b5fad:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-16T14:32:30,197 INFO [regionserver/04e93f2b5fad:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-16T14:32:30,197 INFO [RS:2;04e93f2b5fad:45119 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-16T14:32:30,197 INFO [RS:2;04e93f2b5fad:45119 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-16T14:32:30,197 INFO [RS:2;04e93f2b5fad:45119 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-16T14:32:30,198 INFO [RS:2;04e93f2b5fad:45119 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-16T14:32:30,198 INFO [RS:2;04e93f2b5fad:45119 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:45119 2024-11-16T14:32:30,285 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45119-0x101440f72da0003, quorum=127.0.0.1:57212, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/04e93f2b5fad,45119,1731767533018 2024-11-16T14:32:30,285 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34365-0x101440f72da0000, quorum=127.0.0.1:57212, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-16T14:32:30,285 INFO [RS:2;04e93f2b5fad:45119 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-16T14:32:30,296 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [04e93f2b5fad,45119,1731767533018] 2024-11-16T14:32:30,306 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/04e93f2b5fad,45119,1731767533018 already deleted, retry=false 2024-11-16T14:32:30,306 INFO [RegionServerTracker-0 {}] master.ServerManager(695): Processing expiration of 04e93f2b5fad,45119,1731767533018 on 04e93f2b5fad,34365,1731767532092 2024-11-16T14:32:30,313 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1731767547738.9cc76f2c81b25eccf90da7ca97d0721e., hostname=04e93f2b5fad,45119,1731767533018, seqNum=18] 2024-11-16T14:32:30,316 WARN [RPCClient-NioEventLoopGroup-6-5 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 04e93f2b5fad:45119 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: 04e93f2b5fad/172.17.0.2:45119 Caused by: java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hbase.thirdparty.io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:336) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:339) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:32:30,316 DEBUG [RegionServerTracker-0 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure 04e93f2b5fad,45119,1731767533018, splitWal=true, meta=false 2024-11-16T14:32:30,317 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1731767547738.9cc76f2c81b25eccf90da7ca97d0721e., hostname=04e93f2b5fad,45119,1731767533018, seqNum=18 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1731767547738.9cc76f2c81b25eccf90da7ca97d0721e., hostname=04e93f2b5fad,45119,1731767533018, seqNum=18, error=java.net.ConnectException: Call to address=04e93f2b5fad:45119 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: 04e93f2b5fad/172.17.0.2:45119 2024-11-16T14:32:30,317 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1731767547738.9cc76f2c81b25eccf90da7ca97d0721e., hostname=04e93f2b5fad,45119,1731767533018, seqNum=18 is java.net.ConnectException: Connection refused 2024-11-16T14:32:30,317 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testReplayEditsAfterRegionMovedWithMultiCF,,1731767547738.9cc76f2c81b25eccf90da7ca97d0721e., hostname=04e93f2b5fad,45119,1731767533018, seqNum=18 from cache 2024-11-16T14:32:30,318 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] ipc.FailedServers(52): Added failed server with address 04e93f2b5fad:45119 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: 04e93f2b5fad/172.17.0.2:45119 2024-11-16T14:32:30,319 INFO [RegionServerTracker-0 {}] assignment.AssignmentManager(1991): Scheduled ServerCrashProcedure pid=13 for 04e93f2b5fad,45119,1731767533018 (carryingMeta=false) 04e93f2b5fad,45119,1731767533018/CRASHED/regionCount=1/lock=java.util.concurrent.locks.ReentrantReadWriteLock@357756ff[Write locks = 1, Read locks = 0], oldState=ONLINE. 2024-11-16T14:32:30,319 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(169): Start pid=13, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure 04e93f2b5fad,45119,1731767533018, splitWal=true, meta=false 2024-11-16T14:32:30,321 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(207): 04e93f2b5fad,45119,1731767533018 had 1 regions 2024-11-16T14:32:30,322 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=13, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure 04e93f2b5fad,45119,1731767533018, splitWal=true, meta=false, isMeta: false 2024-11-16T14:32:30,323 DEBUG [PEWorker-2 {}] master.MasterWalManager(329): Renamed region directory: hdfs://localhost:43457/user/jenkins/test-data/0d13f5ec-97ce-91d4-7824-c5174956db56/WALs/04e93f2b5fad,45119,1731767533018-splitting 2024-11-16T14:32:30,325 INFO [PEWorker-2 {}] master.SplitWALManager(105): 04e93f2b5fad,45119,1731767533018 WAL count=1, meta=false 2024-11-16T14:32:30,327 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure 04e93f2b5fad%2C45119%2C1731767533018.1731767534512}] 2024-11-16T14:32:30,332 DEBUG [PEWorker-1 {}] master.SplitWALManager(158): Acquired split WAL worker=04e93f2b5fad,35241,1731767532873 2024-11-16T14:32:30,335 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=15, ppid=14, state=RUNNABLE, hasLock=false; SplitWALRemoteProcedure 04e93f2b5fad%2C45119%2C1731767533018.1731767534512, worker=04e93f2b5fad,35241,1731767532873}] 2024-11-16T14:32:30,396 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45119-0x101440f72da0003, quorum=127.0.0.1:57212, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-16T14:32:30,396 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45119-0x101440f72da0003, quorum=127.0.0.1:57212, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-16T14:32:30,396 INFO [RS:2;04e93f2b5fad:45119 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-16T14:32:30,397 INFO [RS:2;04e93f2b5fad:45119 {}] regionserver.HRegionServer(1031): Exiting; stopping=04e93f2b5fad,45119,1731767533018; zookeeper connection closed. 2024-11-16T14:32:30,397 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@2832d836 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@2832d836 2024-11-16T14:32:30,496 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35241 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SplitWALCallable, pid=15 2024-11-16T14:32:30,512 INFO [RS_LOG_REPLAY_OPS-regionserver/04e93f2b5fad:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(299): Splitting hdfs://localhost:43457/user/jenkins/test-data/0d13f5ec-97ce-91d4-7824-c5174956db56/WALs/04e93f2b5fad,45119,1731767533018-splitting/04e93f2b5fad%2C45119%2C1731767533018.1731767534512, size=1.4 K (1407bytes) 2024-11-16T14:32:30,512 INFO [RS_LOG_REPLAY_OPS-regionserver/04e93f2b5fad:0-0 {event_type=RS_LOG_REPLAY, pid=15}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:43457/user/jenkins/test-data/0d13f5ec-97ce-91d4-7824-c5174956db56/WALs/04e93f2b5fad,45119,1731767533018-splitting/04e93f2b5fad%2C45119%2C1731767533018.1731767534512 2024-11-16T14:32:30,512 INFO [RS_LOG_REPLAY_OPS-regionserver/04e93f2b5fad:0-0 {event_type=RS_LOG_REPLAY, pid=15}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:43457/user/jenkins/test-data/0d13f5ec-97ce-91d4-7824-c5174956db56/WALs/04e93f2b5fad,45119,1731767533018-splitting/04e93f2b5fad%2C45119%2C1731767533018.1731767534512 after 0ms 2024-11-16T14:32:30,515 DEBUG [RS_LOG_REPLAY_OPS-regionserver/04e93f2b5fad:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:43457/user/jenkins/test-data/0d13f5ec-97ce-91d4-7824-c5174956db56/WALs/04e93f2b5fad,45119,1731767533018-splitting/04e93f2b5fad%2C45119%2C1731767533018.1731767534512: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-16T14:32:30,516 INFO [RS_LOG_REPLAY_OPS-regionserver/04e93f2b5fad:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(310): Open hdfs://localhost:43457/user/jenkins/test-data/0d13f5ec-97ce-91d4-7824-c5174956db56/WALs/04e93f2b5fad,45119,1731767533018-splitting/04e93f2b5fad%2C45119%2C1731767533018.1731767534512 took 5ms 2024-11-16T14:32:30,522 DEBUG [RS_LOG_REPLAY_OPS-regionserver/04e93f2b5fad:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(352): Last flushed sequenceid for 9cc76f2c81b25eccf90da7ca97d0721e: last_flushed_sequence_id: 12 store_sequence_id { family_name: "cf1" sequence_id: 12 } store_sequence_id { family_name: "cf2" sequence_id: 12 } 2024-11-16T14:32:30,522 DEBUG [RS_LOG_REPLAY_OPS-regionserver/04e93f2b5fad:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:43457/user/jenkins/test-data/0d13f5ec-97ce-91d4-7824-c5174956db56/WALs/04e93f2b5fad,45119,1731767533018-splitting/04e93f2b5fad%2C45119%2C1731767533018.1731767534512 so closing down 2024-11-16T14:32:30,522 DEBUG [RS_LOG_REPLAY_OPS-regionserver/04e93f2b5fad:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-16T14:32:30,522 INFO [RS_LOG_REPLAY_OPS-regionserver/04e93f2b5fad:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.OutputSink(145): 3 split writer threads finished 2024-11-16T14:32:30,523 INFO [RS_LOG_REPLAY_OPS-regionserver/04e93f2b5fad:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(425): Processed 6 edits across 0 Regions in 6 ms; skipped=6; WAL=hdfs://localhost:43457/user/jenkins/test-data/0d13f5ec-97ce-91d4-7824-c5174956db56/WALs/04e93f2b5fad,45119,1731767533018-splitting/04e93f2b5fad%2C45119%2C1731767533018.1731767534512, size=1.4 K, length=1407, corrupted=false, cancelled=false 2024-11-16T14:32:30,523 DEBUG [RS_LOG_REPLAY_OPS-regionserver/04e93f2b5fad:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(428): Completed split of hdfs://localhost:43457/user/jenkins/test-data/0d13f5ec-97ce-91d4-7824-c5174956db56/WALs/04e93f2b5fad,45119,1731767533018-splitting/04e93f2b5fad%2C45119%2C1731767533018.1731767534512, journal: Splitting hdfs://localhost:43457/user/jenkins/test-data/0d13f5ec-97ce-91d4-7824-c5174956db56/WALs/04e93f2b5fad,45119,1731767533018-splitting/04e93f2b5fad%2C45119%2C1731767533018.1731767534512, size=1.4 K (1407bytes) at 1731767550512Finishing writing output for hdfs://localhost:43457/user/jenkins/test-data/0d13f5ec-97ce-91d4-7824-c5174956db56/WALs/04e93f2b5fad,45119,1731767533018-splitting/04e93f2b5fad%2C45119%2C1731767533018.1731767534512 so closing down at 1731767550522 (+10 ms)3 split writer threads finished at 1731767550522Processed 6 edits across 0 Regions in 6 ms; skipped=6; WAL=hdfs://localhost:43457/user/jenkins/test-data/0d13f5ec-97ce-91d4-7824-c5174956db56/WALs/04e93f2b5fad,45119,1731767533018-splitting/04e93f2b5fad%2C45119%2C1731767533018.1731767534512, size=1.4 K, length=1407, corrupted=false, cancelled=false at 1731767550523 (+1 ms) 2024-11-16T14:32:30,523 DEBUG [RS_LOG_REPLAY_OPS-regionserver/04e93f2b5fad:0-0 {event_type=RS_LOG_REPLAY, pid=15}] regionserver.SplitLogWorker(218): Done splitting WAL hdfs://localhost:43457/user/jenkins/test-data/0d13f5ec-97ce-91d4-7824-c5174956db56/WALs/04e93f2b5fad,45119,1731767533018-splitting/04e93f2b5fad%2C45119%2C1731767533018.1731767534512 2024-11-16T14:32:30,524 DEBUG [RS_LOG_REPLAY_OPS-regionserver/04e93f2b5fad:0-0 {event_type=RS_LOG_REPLAY, pid=15}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=15 2024-11-16T14:32:30,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34365 {}] master.HMaster(4169): Remote procedure done, pid=15 2024-11-16T14:32:30,531 INFO [PEWorker-4 {}] wal.WALSplitUtil(143): Moved hdfs://localhost:43457/user/jenkins/test-data/0d13f5ec-97ce-91d4-7824-c5174956db56/WALs/04e93f2b5fad,45119,1731767533018-splitting/04e93f2b5fad%2C45119%2C1731767533018.1731767534512 to hdfs://localhost:43457/user/jenkins/test-data/0d13f5ec-97ce-91d4-7824-c5174956db56/oldWALs 2024-11-16T14:32:30,535 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=15, resume processing ppid=14 2024-11-16T14:32:30,535 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, ppid=14, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 04e93f2b5fad%2C45119%2C1731767533018.1731767534512, worker=04e93f2b5fad,35241,1731767532873 in 197 msec 2024-11-16T14:32:30,536 DEBUG [PEWorker-5 {}] master.SplitWALManager(172): Release split WAL worker=04e93f2b5fad,35241,1731767532873 2024-11-16T14:32:30,539 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-11-16T14:32:30,540 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; SplitWALProcedure 04e93f2b5fad%2C45119%2C1731767533018.1731767534512, worker=04e93f2b5fad,35241,1731767532873 in 211 msec 2024-11-16T14:32:30,541 INFO [PEWorker-2 {}] master.SplitLogManager(171): hdfs://localhost:43457/user/jenkins/test-data/0d13f5ec-97ce-91d4-7824-c5174956db56/WALs/04e93f2b5fad,45119,1731767533018-splitting dir is empty, no logs to split. 2024-11-16T14:32:30,541 INFO [PEWorker-2 {}] master.SplitWALManager(105): 04e93f2b5fad,45119,1731767533018 WAL count=0, meta=false 2024-11-16T14:32:30,541 DEBUG [PEWorker-2 {}] procedure.ServerCrashProcedure(329): Check if 04e93f2b5fad,45119,1731767533018 WAL splitting is done? wals=0, meta=false 2024-11-16T14:32:30,543 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(321): Remove WAL directory for 04e93f2b5fad,45119,1731767533018 failed, ignore...File hdfs://localhost:43457/user/jenkins/test-data/0d13f5ec-97ce-91d4-7824-c5174956db56/WALs/04e93f2b5fad,45119,1731767533018-splitting does not exist. 2024-11-16T14:32:30,545 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=9cc76f2c81b25eccf90da7ca97d0721e, ASSIGN}] 2024-11-16T14:32:30,547 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=9cc76f2c81b25eccf90da7ca97d0721e, ASSIGN 2024-11-16T14:32:30,548 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=9cc76f2c81b25eccf90da7ca97d0721e, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-16T14:32:30,629 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1731767547738.9cc76f2c81b25eccf90da7ca97d0721e., hostname=04e93f2b5fad,45119,1731767533018, seqNum=18] 2024-11-16T14:32:30,630 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] ipc.AbstractRpcClient(357): Not trying to connect to 04e93f2b5fad:45119 this server is in the failed servers list 2024-11-16T14:32:30,631 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1731767547738.9cc76f2c81b25eccf90da7ca97d0721e., hostname=04e93f2b5fad,45119,1731767533018, seqNum=18 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1731767547738.9cc76f2c81b25eccf90da7ca97d0721e., hostname=04e93f2b5fad,45119,1731767533018, seqNum=18, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=04e93f2b5fad:45119 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 04e93f2b5fad:45119 2024-11-16T14:32:30,631 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1731767547738.9cc76f2c81b25eccf90da7ca97d0721e., hostname=04e93f2b5fad,45119,1731767533018, seqNum=18 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 04e93f2b5fad:45119 2024-11-16T14:32:30,631 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testReplayEditsAfterRegionMovedWithMultiCF,,1731767547738.9cc76f2c81b25eccf90da7ca97d0721e., hostname=04e93f2b5fad,45119,1731767533018, seqNum=18 from cache 2024-11-16T14:32:30,699 DEBUG [04e93f2b5fad:34365 {}] balancer.BalancerClusterState(204): Hosts are {04e93f2b5fad=0} racks are {/default-rack=0} 2024-11-16T14:32:30,699 DEBUG [04e93f2b5fad:34365 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-16T14:32:30,699 DEBUG [04e93f2b5fad:34365 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-16T14:32:30,699 DEBUG [04e93f2b5fad:34365 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-16T14:32:30,699 DEBUG [04e93f2b5fad:34365 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-16T14:32:30,700 INFO [04e93f2b5fad:34365 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-16T14:32:30,700 INFO [04e93f2b5fad:34365 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-16T14:32:30,700 DEBUG [04e93f2b5fad:34365 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-16T14:32:30,700 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=9cc76f2c81b25eccf90da7ca97d0721e, regionState=OPENING, regionLocation=04e93f2b5fad,39921,1731767532970 2024-11-16T14:32:30,705 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=9cc76f2c81b25eccf90da7ca97d0721e, ASSIGN because future has completed 2024-11-16T14:32:30,705 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=17, ppid=16, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9cc76f2c81b25eccf90da7ca97d0721e, server=04e93f2b5fad,39921,1731767532970}] 2024-11-16T14:32:30,860 DEBUG [RSProcedureDispatcher-pool-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-16T14:32:30,863 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39279, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-16T14:32:30,871 INFO [RS_OPEN_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1731767547738.9cc76f2c81b25eccf90da7ca97d0721e. 2024-11-16T14:32:30,871 DEBUG [RS_OPEN_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(7752): Opening region: {ENCODED => 9cc76f2c81b25eccf90da7ca97d0721e, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1731767547738.9cc76f2c81b25eccf90da7ca97d0721e.', STARTKEY => '', ENDKEY => ''} 2024-11-16T14:32:30,872 DEBUG [RS_OPEN_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF 9cc76f2c81b25eccf90da7ca97d0721e 2024-11-16T14:32:30,872 DEBUG [RS_OPEN_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1731767547738.9cc76f2c81b25eccf90da7ca97d0721e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T14:32:30,872 DEBUG [RS_OPEN_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(7794): checking encryption for 9cc76f2c81b25eccf90da7ca97d0721e 2024-11-16T14:32:30,872 DEBUG [RS_OPEN_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(7797): checking classloading for 9cc76f2c81b25eccf90da7ca97d0721e 2024-11-16T14:32:30,874 INFO [StoreOpener-9cc76f2c81b25eccf90da7ca97d0721e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 9cc76f2c81b25eccf90da7ca97d0721e 2024-11-16T14:32:30,875 INFO [StoreOpener-9cc76f2c81b25eccf90da7ca97d0721e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9cc76f2c81b25eccf90da7ca97d0721e columnFamilyName cf1 2024-11-16T14:32:30,875 DEBUG [StoreOpener-9cc76f2c81b25eccf90da7ca97d0721e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:32:30,881 DEBUG [StoreOpener-9cc76f2c81b25eccf90da7ca97d0721e-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:43457/user/jenkins/test-data/0d13f5ec-97ce-91d4-7824-c5174956db56/data/default/testReplayEditsAfterRegionMovedWithMultiCF/9cc76f2c81b25eccf90da7ca97d0721e/cf1/455896f41041401abaf9c12db5d9175a 2024-11-16T14:32:30,881 INFO [StoreOpener-9cc76f2c81b25eccf90da7ca97d0721e-1 {}] regionserver.HStore(327): Store=9cc76f2c81b25eccf90da7ca97d0721e/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T14:32:30,881 INFO [StoreOpener-9cc76f2c81b25eccf90da7ca97d0721e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 9cc76f2c81b25eccf90da7ca97d0721e 2024-11-16T14:32:30,882 INFO [StoreOpener-9cc76f2c81b25eccf90da7ca97d0721e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9cc76f2c81b25eccf90da7ca97d0721e columnFamilyName cf2 2024-11-16T14:32:30,882 DEBUG [StoreOpener-9cc76f2c81b25eccf90da7ca97d0721e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:32:30,889 DEBUG [StoreOpener-9cc76f2c81b25eccf90da7ca97d0721e-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:43457/user/jenkins/test-data/0d13f5ec-97ce-91d4-7824-c5174956db56/data/default/testReplayEditsAfterRegionMovedWithMultiCF/9cc76f2c81b25eccf90da7ca97d0721e/cf2/1f48dc45b7ab43a29916ce920150edec 2024-11-16T14:32:30,889 INFO [StoreOpener-9cc76f2c81b25eccf90da7ca97d0721e-1 {}] regionserver.HStore(327): Store=9cc76f2c81b25eccf90da7ca97d0721e/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T14:32:30,889 DEBUG [RS_OPEN_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1038): replaying wal for 9cc76f2c81b25eccf90da7ca97d0721e 2024-11-16T14:32:30,890 DEBUG [RS_OPEN_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43457/user/jenkins/test-data/0d13f5ec-97ce-91d4-7824-c5174956db56/data/default/testReplayEditsAfterRegionMovedWithMultiCF/9cc76f2c81b25eccf90da7ca97d0721e 2024-11-16T14:32:30,891 DEBUG [RS_OPEN_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43457/user/jenkins/test-data/0d13f5ec-97ce-91d4-7824-c5174956db56/data/default/testReplayEditsAfterRegionMovedWithMultiCF/9cc76f2c81b25eccf90da7ca97d0721e 2024-11-16T14:32:30,892 DEBUG [RS_OPEN_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1048): stopping wal replay for 9cc76f2c81b25eccf90da7ca97d0721e 2024-11-16T14:32:30,892 DEBUG [RS_OPEN_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1060): Cleaning up temporary data for 9cc76f2c81b25eccf90da7ca97d0721e 2024-11-16T14:32:30,893 DEBUG [RS_OPEN_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-11-16T14:32:30,894 DEBUG [RS_OPEN_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1093): writing seq id for 9cc76f2c81b25eccf90da7ca97d0721e 2024-11-16T14:32:30,895 INFO [RS_OPEN_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1114): Opened 9cc76f2c81b25eccf90da7ca97d0721e; next sequenceid=18; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65378265, jitterRate=-0.02578793466091156}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-11-16T14:32:30,895 DEBUG [RS_OPEN_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 9cc76f2c81b25eccf90da7ca97d0721e 2024-11-16T14:32:30,896 DEBUG [RS_OPEN_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1006): Region open journal for 9cc76f2c81b25eccf90da7ca97d0721e: Running coprocessor pre-open hook at 1731767550872Writing region info on filesystem at 1731767550873 (+1 ms)Initializing all the Stores at 1731767550873Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731767550874 (+1 ms)Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731767550874Cleaning up temporary data from old regions at 1731767550892 (+18 ms)Running coprocessor post-open hooks at 1731767550895 (+3 ms)Region opened successfully at 1731767550896 (+1 ms) 2024-11-16T14:32:30,897 INFO [RS_OPEN_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1731767547738.9cc76f2c81b25eccf90da7ca97d0721e., pid=17, masterSystemTime=1731767550860 2024-11-16T14:32:30,900 DEBUG [RS_OPEN_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1731767547738.9cc76f2c81b25eccf90da7ca97d0721e. 2024-11-16T14:32:30,900 INFO [RS_OPEN_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1731767547738.9cc76f2c81b25eccf90da7ca97d0721e. 2024-11-16T14:32:30,900 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=9cc76f2c81b25eccf90da7ca97d0721e, regionState=OPEN, openSeqNum=18, regionLocation=04e93f2b5fad,39921,1731767532970 2024-11-16T14:32:30,903 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=17, ppid=16, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9cc76f2c81b25eccf90da7ca97d0721e, server=04e93f2b5fad,39921,1731767532970 because future has completed 2024-11-16T14:32:30,906 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=17, resume processing ppid=16 2024-11-16T14:32:30,907 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, ppid=16, state=SUCCESS, hasLock=false; OpenRegionProcedure 9cc76f2c81b25eccf90da7ca97d0721e, server=04e93f2b5fad,39921,1731767532970 in 199 msec 2024-11-16T14:32:30,909 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=13 2024-11-16T14:32:30,909 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=13, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=9cc76f2c81b25eccf90da7ca97d0721e, ASSIGN in 361 msec 2024-11-16T14:32:30,909 INFO [PEWorker-4 {}] procedure.ServerCrashProcedure(291): removed crashed server 04e93f2b5fad,45119,1731767533018 after splitting done 2024-11-16T14:32:30,912 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; ServerCrashProcedure 04e93f2b5fad,45119,1731767533018, splitWal=true, meta=false in 601 msec 2024-11-16T14:32:31,147 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1731767547738.9cc76f2c81b25eccf90da7ca97d0721e., hostname=04e93f2b5fad,39921,1731767532970, seqNum=18] 2024-11-16T14:32:31,148 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T14:32:31,150 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47276, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T14:32:31,167 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayValueCompression#testReplayEditsAfterRegionMovedWithMultiCF Thread=410 (was 411), OpenFileDescriptor=1051 (was 1023) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=168 (was 168), ProcessCount=11 (was 11), AvailableMemoryMB=4513 (was 4518) 2024-11-16T14:32:31,169 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1051 is superior to 1024 2024-11-16T14:32:31,183 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayValueCompression#testReplayEditsAfterPartialFlush Thread=410, OpenFileDescriptor=1051, MaxFileDescriptor=1048576, SystemLoadAverage=168, ProcessCount=11, AvailableMemoryMB=4513 2024-11-16T14:32:31,183 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1051 is superior to 1024 2024-11-16T14:32:31,199 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-16T14:32:31,201 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-16T14:32:31,202 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-16T14:32:31,205 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-18233897, suffix=, logDir=hdfs://localhost:43457/hbase/WALs/hregion-18233897, archiveDir=hdfs://localhost:43457/hbase/oldWALs, maxLogs=32 2024-11-16T14:32:31,223 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-18233897/hregion-18233897.1731767551205, exclude list is [], retry=0 2024-11-16T14:32:31,226 DEBUG [AsyncFSWAL-20-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43279,DS-17ab9554-4509-49b3-918b-178692b9a473,DISK] 2024-11-16T14:32:31,226 DEBUG [AsyncFSWAL-20-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36193,DS-a30e140c-8532-4096-8cae-c8d713a3d4e3,DISK] 2024-11-16T14:32:31,226 DEBUG [AsyncFSWAL-20-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33005,DS-6a1dd528-c0be-4580-a53e-7113879f7f97,DISK] 2024-11-16T14:32:31,228 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-18233897/hregion-18233897.1731767551205 2024-11-16T14:32:31,228 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36015:36015),(127.0.0.1/127.0.0.1:35677:35677),(127.0.0.1/127.0.0.1:41179:41179)] 2024-11-16T14:32:31,228 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 4f290312a7822e6e1180a0e3c454fd9f, NAME => 'testReplayEditsWrittenViaHRegion,,1731767551200.4f290312a7822e6e1180a0e3c454fd9f.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenViaHRegion', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43457/hbase 2024-11-16T14:32:31,240 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43279 is added to blk_1073741893_1071 (size=67) 2024-11-16T14:32:31,240 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36193 is added to blk_1073741893_1071 (size=67) 2024-11-16T14:32:31,240 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33005 is added to blk_1073741893_1071 (size=67) 2024-11-16T14:32:31,241 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1731767551200.4f290312a7822e6e1180a0e3c454fd9f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T14:32:31,242 INFO [StoreOpener-4f290312a7822e6e1180a0e3c454fd9f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 4f290312a7822e6e1180a0e3c454fd9f 2024-11-16T14:32:31,243 INFO [StoreOpener-4f290312a7822e6e1180a0e3c454fd9f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4f290312a7822e6e1180a0e3c454fd9f columnFamilyName a 2024-11-16T14:32:31,244 DEBUG [StoreOpener-4f290312a7822e6e1180a0e3c454fd9f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:32:31,244 INFO [StoreOpener-4f290312a7822e6e1180a0e3c454fd9f-1 {}] regionserver.HStore(327): Store=4f290312a7822e6e1180a0e3c454fd9f/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T14:32:31,244 INFO [StoreOpener-4f290312a7822e6e1180a0e3c454fd9f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 4f290312a7822e6e1180a0e3c454fd9f 2024-11-16T14:32:31,245 INFO [StoreOpener-4f290312a7822e6e1180a0e3c454fd9f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4f290312a7822e6e1180a0e3c454fd9f columnFamilyName b 2024-11-16T14:32:31,245 DEBUG [StoreOpener-4f290312a7822e6e1180a0e3c454fd9f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:32:31,246 INFO [StoreOpener-4f290312a7822e6e1180a0e3c454fd9f-1 {}] regionserver.HStore(327): Store=4f290312a7822e6e1180a0e3c454fd9f/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T14:32:31,246 INFO [StoreOpener-4f290312a7822e6e1180a0e3c454fd9f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 4f290312a7822e6e1180a0e3c454fd9f 2024-11-16T14:32:31,247 INFO [StoreOpener-4f290312a7822e6e1180a0e3c454fd9f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4f290312a7822e6e1180a0e3c454fd9f columnFamilyName c 2024-11-16T14:32:31,247 DEBUG [StoreOpener-4f290312a7822e6e1180a0e3c454fd9f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:32:31,248 INFO [StoreOpener-4f290312a7822e6e1180a0e3c454fd9f-1 {}] regionserver.HStore(327): Store=4f290312a7822e6e1180a0e3c454fd9f/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T14:32:31,248 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 4f290312a7822e6e1180a0e3c454fd9f 2024-11-16T14:32:31,249 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43457/hbase/data/default/testReplayEditsWrittenViaHRegion/4f290312a7822e6e1180a0e3c454fd9f 2024-11-16T14:32:31,249 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43457/hbase/data/default/testReplayEditsWrittenViaHRegion/4f290312a7822e6e1180a0e3c454fd9f 2024-11-16T14:32:31,250 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 4f290312a7822e6e1180a0e3c454fd9f 2024-11-16T14:32:31,250 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 4f290312a7822e6e1180a0e3c454fd9f 2024-11-16T14:32:31,251 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-16T14:32:31,252 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 4f290312a7822e6e1180a0e3c454fd9f 2024-11-16T14:32:31,254 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43457/hbase/data/default/testReplayEditsWrittenViaHRegion/4f290312a7822e6e1180a0e3c454fd9f/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-16T14:32:31,254 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 4f290312a7822e6e1180a0e3c454fd9f; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71301642, jitterRate=0.06247726082801819}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-16T14:32:31,255 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 4f290312a7822e6e1180a0e3c454fd9f: Writing region info on filesystem at 1731767551241Initializing all the Stores at 1731767551242 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731767551242Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731767551242Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731767551242Cleaning up temporary data from old regions at 1731767551250 (+8 ms)Region opened successfully at 1731767551255 (+5 ms) 2024-11-16T14:32:31,255 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 4f290312a7822e6e1180a0e3c454fd9f, disabling compactions & flushes 2024-11-16T14:32:31,255 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1731767551200.4f290312a7822e6e1180a0e3c454fd9f. 2024-11-16T14:32:31,255 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1731767551200.4f290312a7822e6e1180a0e3c454fd9f. 2024-11-16T14:32:31,255 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1731767551200.4f290312a7822e6e1180a0e3c454fd9f. after waiting 0 ms 2024-11-16T14:32:31,255 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1731767551200.4f290312a7822e6e1180a0e3c454fd9f. 2024-11-16T14:32:31,256 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1731767551200.4f290312a7822e6e1180a0e3c454fd9f. 2024-11-16T14:32:31,256 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 4f290312a7822e6e1180a0e3c454fd9f: Waiting for close lock at 1731767551255Disabling compacts and flushes for region at 1731767551255Disabling writes for close at 1731767551255Writing region close event to WAL at 1731767551256 (+1 ms)Closed at 1731767551256 2024-11-16T14:32:31,258 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43279 is added to blk_1073741892_1070 (size=95) 2024-11-16T14:32:31,258 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33005 is added to blk_1073741892_1070 (size=95) 2024-11-16T14:32:31,258 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36193 is added to blk_1073741892_1070 (size=95) 2024-11-16T14:32:31,261 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-16T14:32:31,261 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-18233897:(num 1731767551205) 2024-11-16T14:32:31,261 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-16T14:32:31,264 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:43457/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731767551199, archiveDir=hdfs://localhost:43457/hbase/oldWALs, maxLogs=32 2024-11-16T14:32:31,276 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731767551199/wal.1731767551264, exclude list is [], retry=0 2024-11-16T14:32:31,279 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36193,DS-a30e140c-8532-4096-8cae-c8d713a3d4e3,DISK] 2024-11-16T14:32:31,279 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43279,DS-17ab9554-4509-49b3-918b-178692b9a473,DISK] 2024-11-16T14:32:31,279 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33005,DS-6a1dd528-c0be-4580-a53e-7113879f7f97,DISK] 2024-11-16T14:32:31,281 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731767551199/wal.1731767551264 2024-11-16T14:32:31,281 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35677:35677),(127.0.0.1/127.0.0.1:36015:36015),(127.0.0.1/127.0.0.1:41179:41179)] 2024-11-16T14:32:31,281 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 4f290312a7822e6e1180a0e3c454fd9f, NAME => 'testReplayEditsWrittenViaHRegion,,1731767551200.4f290312a7822e6e1180a0e3c454fd9f.', STARTKEY => '', ENDKEY => ''} 2024-11-16T14:32:31,281 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1731767551200.4f290312a7822e6e1180a0e3c454fd9f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T14:32:31,282 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 4f290312a7822e6e1180a0e3c454fd9f 2024-11-16T14:32:31,282 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 4f290312a7822e6e1180a0e3c454fd9f 2024-11-16T14:32:31,283 INFO [StoreOpener-4f290312a7822e6e1180a0e3c454fd9f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 4f290312a7822e6e1180a0e3c454fd9f 2024-11-16T14:32:31,284 INFO [StoreOpener-4f290312a7822e6e1180a0e3c454fd9f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4f290312a7822e6e1180a0e3c454fd9f columnFamilyName a 2024-11-16T14:32:31,284 DEBUG [StoreOpener-4f290312a7822e6e1180a0e3c454fd9f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:32:31,284 INFO [StoreOpener-4f290312a7822e6e1180a0e3c454fd9f-1 {}] regionserver.HStore(327): Store=4f290312a7822e6e1180a0e3c454fd9f/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T14:32:31,284 INFO [StoreOpener-4f290312a7822e6e1180a0e3c454fd9f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 4f290312a7822e6e1180a0e3c454fd9f 2024-11-16T14:32:31,285 INFO [StoreOpener-4f290312a7822e6e1180a0e3c454fd9f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4f290312a7822e6e1180a0e3c454fd9f columnFamilyName b 2024-11-16T14:32:31,285 DEBUG [StoreOpener-4f290312a7822e6e1180a0e3c454fd9f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:32:31,285 INFO [StoreOpener-4f290312a7822e6e1180a0e3c454fd9f-1 {}] regionserver.HStore(327): Store=4f290312a7822e6e1180a0e3c454fd9f/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T14:32:31,285 INFO [StoreOpener-4f290312a7822e6e1180a0e3c454fd9f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 4f290312a7822e6e1180a0e3c454fd9f 2024-11-16T14:32:31,286 INFO [StoreOpener-4f290312a7822e6e1180a0e3c454fd9f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4f290312a7822e6e1180a0e3c454fd9f columnFamilyName c 2024-11-16T14:32:31,286 DEBUG [StoreOpener-4f290312a7822e6e1180a0e3c454fd9f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:32:31,287 INFO [StoreOpener-4f290312a7822e6e1180a0e3c454fd9f-1 {}] regionserver.HStore(327): Store=4f290312a7822e6e1180a0e3c454fd9f/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T14:32:31,287 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 4f290312a7822e6e1180a0e3c454fd9f 2024-11-16T14:32:31,287 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43457/hbase/data/default/testReplayEditsWrittenViaHRegion/4f290312a7822e6e1180a0e3c454fd9f 2024-11-16T14:32:31,288 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43457/hbase/data/default/testReplayEditsWrittenViaHRegion/4f290312a7822e6e1180a0e3c454fd9f 2024-11-16T14:32:31,289 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 4f290312a7822e6e1180a0e3c454fd9f 2024-11-16T14:32:31,290 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 4f290312a7822e6e1180a0e3c454fd9f 2024-11-16T14:32:31,290 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-16T14:32:31,291 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 4f290312a7822e6e1180a0e3c454fd9f 2024-11-16T14:32:31,292 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 4f290312a7822e6e1180a0e3c454fd9f; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62477981, jitterRate=-0.06900553405284882}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-16T14:32:31,292 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 4f290312a7822e6e1180a0e3c454fd9f: Writing region info on filesystem at 1731767551282Initializing all the Stores at 1731767551282Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731767551283 (+1 ms)Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731767551283Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731767551283Cleaning up temporary data from old regions at 1731767551290 (+7 ms)Region opened successfully at 1731767551292 (+2 ms) 2024-11-16T14:32:31,316 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 4f290312a7822e6e1180a0e3c454fd9f 3/3 column families, dataSize=2.55 KB heapSize=5.44 KB 2024-11-16T14:32:31,331 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43457/hbase/data/default/testReplayEditsWrittenViaHRegion/4f290312a7822e6e1180a0e3c454fd9f/.tmp/a/b3f65065406f4bc39f3526823df38a20 is 91, key is testReplayEditsWrittenViaHRegion/a:x0/1731767551292/Put/seqid=0 2024-11-16T14:32:31,338 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36193 is added to blk_1073741895_1073 (size=5958) 2024-11-16T14:32:31,338 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43279 is added to blk_1073741895_1073 (size=5958) 2024-11-16T14:32:31,338 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33005 is added to blk_1073741895_1073 (size=5958) 2024-11-16T14:32:31,340 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:43457/hbase/data/default/testReplayEditsWrittenViaHRegion/4f290312a7822e6e1180a0e3c454fd9f/.tmp/a/b3f65065406f4bc39f3526823df38a20 2024-11-16T14:32:31,365 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43457/hbase/data/default/testReplayEditsWrittenViaHRegion/4f290312a7822e6e1180a0e3c454fd9f/.tmp/b/bc53e7394eba41e087dbf113a3846f09 is 91, key is testReplayEditsWrittenViaHRegion/b:x0/1731767551299/Put/seqid=0 2024-11-16T14:32:31,371 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36193 is added to blk_1073741896_1074 (size=5958) 2024-11-16T14:32:31,371 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33005 is added to blk_1073741896_1074 (size=5958) 2024-11-16T14:32:31,372 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43279 is added to blk_1073741896_1074 (size=5958) 2024-11-16T14:32:31,372 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:43457/hbase/data/default/testReplayEditsWrittenViaHRegion/4f290312a7822e6e1180a0e3c454fd9f/.tmp/b/bc53e7394eba41e087dbf113a3846f09 2024-11-16T14:32:31,392 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43457/hbase/data/default/testReplayEditsWrittenViaHRegion/4f290312a7822e6e1180a0e3c454fd9f/.tmp/c/25c38840bd464b95b6a58675296d6e8f is 91, key is testReplayEditsWrittenViaHRegion/c:x0/1731767551306/Put/seqid=0 2024-11-16T14:32:31,402 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33005 is added to blk_1073741897_1075 (size=5958) 2024-11-16T14:32:31,402 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36193 is added to blk_1073741897_1075 (size=5958) 2024-11-16T14:32:31,402 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43279 is added to blk_1073741897_1075 (size=5958) 2024-11-16T14:32:31,403 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:43457/hbase/data/default/testReplayEditsWrittenViaHRegion/4f290312a7822e6e1180a0e3c454fd9f/.tmp/c/25c38840bd464b95b6a58675296d6e8f 2024-11-16T14:32:31,410 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43457/hbase/data/default/testReplayEditsWrittenViaHRegion/4f290312a7822e6e1180a0e3c454fd9f/.tmp/a/b3f65065406f4bc39f3526823df38a20 as hdfs://localhost:43457/hbase/data/default/testReplayEditsWrittenViaHRegion/4f290312a7822e6e1180a0e3c454fd9f/a/b3f65065406f4bc39f3526823df38a20 2024-11-16T14:32:31,418 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43457/hbase/data/default/testReplayEditsWrittenViaHRegion/4f290312a7822e6e1180a0e3c454fd9f/a/b3f65065406f4bc39f3526823df38a20, entries=10, sequenceid=33, filesize=5.8 K 2024-11-16T14:32:31,419 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43457/hbase/data/default/testReplayEditsWrittenViaHRegion/4f290312a7822e6e1180a0e3c454fd9f/.tmp/b/bc53e7394eba41e087dbf113a3846f09 as hdfs://localhost:43457/hbase/data/default/testReplayEditsWrittenViaHRegion/4f290312a7822e6e1180a0e3c454fd9f/b/bc53e7394eba41e087dbf113a3846f09 2024-11-16T14:32:31,426 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43457/hbase/data/default/testReplayEditsWrittenViaHRegion/4f290312a7822e6e1180a0e3c454fd9f/b/bc53e7394eba41e087dbf113a3846f09, entries=10, sequenceid=33, filesize=5.8 K 2024-11-16T14:32:31,428 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43457/hbase/data/default/testReplayEditsWrittenViaHRegion/4f290312a7822e6e1180a0e3c454fd9f/.tmp/c/25c38840bd464b95b6a58675296d6e8f as hdfs://localhost:43457/hbase/data/default/testReplayEditsWrittenViaHRegion/4f290312a7822e6e1180a0e3c454fd9f/c/25c38840bd464b95b6a58675296d6e8f 2024-11-16T14:32:31,434 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43457/hbase/data/default/testReplayEditsWrittenViaHRegion/4f290312a7822e6e1180a0e3c454fd9f/c/25c38840bd464b95b6a58675296d6e8f, entries=10, sequenceid=33, filesize=5.8 K 2024-11-16T14:32:31,436 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~2.55 KB/2610, heapSize ~5.39 KB/5520, currentSize=0 B/0 for 4f290312a7822e6e1180a0e3c454fd9f in 120ms, sequenceid=33, compaction requested=false 2024-11-16T14:32:31,436 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 4f290312a7822e6e1180a0e3c454fd9f: 2024-11-16T14:32:31,436 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 4f290312a7822e6e1180a0e3c454fd9f, disabling compactions & flushes 2024-11-16T14:32:31,436 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1731767551200.4f290312a7822e6e1180a0e3c454fd9f. 2024-11-16T14:32:31,436 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1731767551200.4f290312a7822e6e1180a0e3c454fd9f. 2024-11-16T14:32:31,436 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1731767551200.4f290312a7822e6e1180a0e3c454fd9f. after waiting 0 ms 2024-11-16T14:32:31,436 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1731767551200.4f290312a7822e6e1180a0e3c454fd9f. 2024-11-16T14:32:31,438 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1731767551200.4f290312a7822e6e1180a0e3c454fd9f. 2024-11-16T14:32:31,438 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 4f290312a7822e6e1180a0e3c454fd9f: Waiting for close lock at 1731767551436Disabling compacts and flushes for region at 1731767551436Disabling writes for close at 1731767551436Writing region close event to WAL at 1731767551438 (+2 ms)Closed at 1731767551438 2024-11-16T14:32:31,441 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731767551199/wal.1731767551264 not finished, retry = 0 2024-11-16T14:32:31,443 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36193 is added to blk_1073741894_1072 (size=3385) 2024-11-16T14:32:31,443 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33005 is added to blk_1073741894_1072 (size=3385) 2024-11-16T14:32:31,443 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43279 is added to blk_1073741894_1072 (size=3385) 2024-11-16T14:32:31,548 DEBUG [Time-limited test {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43457/hbase/data/default/testReplayEditsWrittenViaHRegion/4f290312a7822e6e1180a0e3c454fd9f/b/bc53e7394eba41e087dbf113a3846f09 to hdfs://localhost:43457/hbase/archive/data/default/testReplayEditsWrittenViaHRegion/4f290312a7822e6e1180a0e3c454fd9f/b/bc53e7394eba41e087dbf113a3846f09 2024-11-16T14:32:31,566 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:43457/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731767551199/wal.1731767551264, size=3.3 K (3385bytes) 2024-11-16T14:32:31,566 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:43457/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731767551199/wal.1731767551264 2024-11-16T14:32:31,567 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:43457/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731767551199/wal.1731767551264 after 1ms 2024-11-16T14:32:31,569 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:43457/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731767551199/wal.1731767551264: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-16T14:32:31,569 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:43457/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731767551199/wal.1731767551264 took 3ms 2024-11-16T14:32:31,572 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:43457/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731767551199/wal.1731767551264 so closing down 2024-11-16T14:32:31,572 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-16T14:32:31,573 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1731767551264.temp 2024-11-16T14:32:31,574 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:43457/hbase/data/default/testReplayEditsWrittenViaHRegion/4f290312a7822e6e1180a0e3c454fd9f/recovered.edits/0000000000000000003-wal.1731767551264.temp 2024-11-16T14:32:31,575 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-16T14:32:31,581 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33005 is added to blk_1073741898_1076 (size=2944) 2024-11-16T14:32:31,582 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43279 is added to blk_1073741898_1076 (size=2944) 2024-11-16T14:32:31,582 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36193 is added to blk_1073741898_1076 (size=2944) 2024-11-16T14:32:31,582 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:43457/hbase/data/default/testReplayEditsWrittenViaHRegion/4f290312a7822e6e1180a0e3c454fd9f/recovered.edits/0000000000000000003-wal.1731767551264.temp (wrote 30 edits, skipped 0 edits in 0 ms) 2024-11-16T14:32:31,584 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:43457/hbase/data/default/testReplayEditsWrittenViaHRegion/4f290312a7822e6e1180a0e3c454fd9f/recovered.edits/0000000000000000003-wal.1731767551264.temp to hdfs://localhost:43457/hbase/data/default/testReplayEditsWrittenViaHRegion/4f290312a7822e6e1180a0e3c454fd9f/recovered.edits/0000000000000000032 2024-11-16T14:32:31,584 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 32 edits across 1 Regions in 14 ms; skipped=2; WAL=hdfs://localhost:43457/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731767551199/wal.1731767551264, size=3.3 K, length=3385, corrupted=false, cancelled=false 2024-11-16T14:32:31,584 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:43457/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731767551199/wal.1731767551264, journal: Splitting hdfs://localhost:43457/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731767551199/wal.1731767551264, size=3.3 K (3385bytes) at 1731767551566Finishing writing output for hdfs://localhost:43457/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731767551199/wal.1731767551264 so closing down at 1731767551572 (+6 ms)Creating recovered edits writer path=hdfs://localhost:43457/hbase/data/default/testReplayEditsWrittenViaHRegion/4f290312a7822e6e1180a0e3c454fd9f/recovered.edits/0000000000000000003-wal.1731767551264.temp at 1731767551574 (+2 ms)3 split writer threads finished at 1731767551575 (+1 ms)Closed recovered edits writer path=hdfs://localhost:43457/hbase/data/default/testReplayEditsWrittenViaHRegion/4f290312a7822e6e1180a0e3c454fd9f/recovered.edits/0000000000000000003-wal.1731767551264.temp (wrote 30 edits, skipped 0 edits in 0 ms) at 1731767551582 (+7 ms)Rename recovered edits hdfs://localhost:43457/hbase/data/default/testReplayEditsWrittenViaHRegion/4f290312a7822e6e1180a0e3c454fd9f/recovered.edits/0000000000000000003-wal.1731767551264.temp to hdfs://localhost:43457/hbase/data/default/testReplayEditsWrittenViaHRegion/4f290312a7822e6e1180a0e3c454fd9f/recovered.edits/0000000000000000032 at 1731767551584 (+2 ms)Processed 32 edits across 1 Regions in 14 ms; skipped=2; WAL=hdfs://localhost:43457/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731767551199/wal.1731767551264, size=3.3 K, length=3385, corrupted=false, cancelled=false at 1731767551584 2024-11-16T14:32:31,586 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:43457/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731767551199/wal.1731767551264 to hdfs://localhost:43457/hbase/oldWALs/wal.1731767551264 2024-11-16T14:32:31,587 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:43457/hbase/data/default/testReplayEditsWrittenViaHRegion/4f290312a7822e6e1180a0e3c454fd9f/recovered.edits/0000000000000000032 2024-11-16T14:32:31,588 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-16T14:32:31,590 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:43457/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731767551199, archiveDir=hdfs://localhost:43457/hbase/oldWALs, maxLogs=32 2024-11-16T14:32:31,609 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731767551199/wal.1731767551590, exclude list is [], retry=0 2024-11-16T14:32:31,612 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36193,DS-a30e140c-8532-4096-8cae-c8d713a3d4e3,DISK] 2024-11-16T14:32:31,613 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33005,DS-6a1dd528-c0be-4580-a53e-7113879f7f97,DISK] 2024-11-16T14:32:31,613 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43279,DS-17ab9554-4509-49b3-918b-178692b9a473,DISK] 2024-11-16T14:32:31,615 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731767551199/wal.1731767551590 2024-11-16T14:32:31,615 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35677:35677),(127.0.0.1/127.0.0.1:41179:41179),(127.0.0.1/127.0.0.1:36015:36015)] 2024-11-16T14:32:31,616 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 4f290312a7822e6e1180a0e3c454fd9f, NAME => 'testReplayEditsWrittenViaHRegion,,1731767551200.4f290312a7822e6e1180a0e3c454fd9f.', STARTKEY => '', ENDKEY => ''} 2024-11-16T14:32:31,616 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1731767551200.4f290312a7822e6e1180a0e3c454fd9f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T14:32:31,616 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 4f290312a7822e6e1180a0e3c454fd9f 2024-11-16T14:32:31,616 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 4f290312a7822e6e1180a0e3c454fd9f 2024-11-16T14:32:31,618 INFO [StoreOpener-4f290312a7822e6e1180a0e3c454fd9f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 4f290312a7822e6e1180a0e3c454fd9f 2024-11-16T14:32:31,619 INFO [StoreOpener-4f290312a7822e6e1180a0e3c454fd9f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4f290312a7822e6e1180a0e3c454fd9f columnFamilyName a 2024-11-16T14:32:31,619 DEBUG [StoreOpener-4f290312a7822e6e1180a0e3c454fd9f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:32:31,625 DEBUG [StoreOpener-4f290312a7822e6e1180a0e3c454fd9f-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:43457/hbase/data/default/testReplayEditsWrittenViaHRegion/4f290312a7822e6e1180a0e3c454fd9f/a/b3f65065406f4bc39f3526823df38a20 2024-11-16T14:32:31,625 INFO [StoreOpener-4f290312a7822e6e1180a0e3c454fd9f-1 {}] regionserver.HStore(327): Store=4f290312a7822e6e1180a0e3c454fd9f/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T14:32:31,625 INFO [StoreOpener-4f290312a7822e6e1180a0e3c454fd9f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 4f290312a7822e6e1180a0e3c454fd9f 2024-11-16T14:32:31,627 INFO [StoreOpener-4f290312a7822e6e1180a0e3c454fd9f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4f290312a7822e6e1180a0e3c454fd9f columnFamilyName b 2024-11-16T14:32:31,627 DEBUG [StoreOpener-4f290312a7822e6e1180a0e3c454fd9f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:32:31,628 INFO [StoreOpener-4f290312a7822e6e1180a0e3c454fd9f-1 {}] regionserver.HStore(327): Store=4f290312a7822e6e1180a0e3c454fd9f/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T14:32:31,628 INFO [StoreOpener-4f290312a7822e6e1180a0e3c454fd9f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 4f290312a7822e6e1180a0e3c454fd9f 2024-11-16T14:32:31,628 INFO [StoreOpener-4f290312a7822e6e1180a0e3c454fd9f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4f290312a7822e6e1180a0e3c454fd9f columnFamilyName c 2024-11-16T14:32:31,629 DEBUG [StoreOpener-4f290312a7822e6e1180a0e3c454fd9f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:32:31,634 DEBUG [StoreOpener-4f290312a7822e6e1180a0e3c454fd9f-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:43457/hbase/data/default/testReplayEditsWrittenViaHRegion/4f290312a7822e6e1180a0e3c454fd9f/c/25c38840bd464b95b6a58675296d6e8f 2024-11-16T14:32:31,634 INFO [StoreOpener-4f290312a7822e6e1180a0e3c454fd9f-1 {}] regionserver.HStore(327): Store=4f290312a7822e6e1180a0e3c454fd9f/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T14:32:31,635 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 4f290312a7822e6e1180a0e3c454fd9f 2024-11-16T14:32:31,636 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43457/hbase/data/default/testReplayEditsWrittenViaHRegion/4f290312a7822e6e1180a0e3c454fd9f 2024-11-16T14:32:31,637 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:43457/hbase/data/default/testReplayEditsWrittenViaHRegion/4f290312a7822e6e1180a0e3c454fd9f 2024-11-16T14:32:31,638 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:43457/hbase/data/default/testReplayEditsWrittenViaHRegion/4f290312a7822e6e1180a0e3c454fd9f/recovered.edits/0000000000000000032 2024-11-16T14:32:31,640 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:43457/hbase/data/default/testReplayEditsWrittenViaHRegion/4f290312a7822e6e1180a0e3c454fd9f/recovered.edits/0000000000000000032: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-16T14:32:31,642 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 10, skipped 20, firstSequenceIdInLog=3, maxSequenceIdInLog=32, path=hdfs://localhost:43457/hbase/data/default/testReplayEditsWrittenViaHRegion/4f290312a7822e6e1180a0e3c454fd9f/recovered.edits/0000000000000000032 2024-11-16T14:32:31,642 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 4f290312a7822e6e1180a0e3c454fd9f 3/3 column families, dataSize=870 B heapSize=2.31 KB 2024-11-16T14:32:31,656 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43457/hbase/data/default/testReplayEditsWrittenViaHRegion/4f290312a7822e6e1180a0e3c454fd9f/.tmp/b/613a3c8ddbf846f6861dac177a93b221 is 91, key is testReplayEditsWrittenViaHRegion/b:x0/1731767551299/Put/seqid=0 2024-11-16T14:32:31,663 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33005 is added to blk_1073741900_1078 (size=5958) 2024-11-16T14:32:31,663 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36193 is added to blk_1073741900_1078 (size=5958) 2024-11-16T14:32:31,663 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43279 is added to blk_1073741900_1078 (size=5958) 2024-11-16T14:32:31,664 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=32 (bloomFilter=true), to=hdfs://localhost:43457/hbase/data/default/testReplayEditsWrittenViaHRegion/4f290312a7822e6e1180a0e3c454fd9f/.tmp/b/613a3c8ddbf846f6861dac177a93b221 2024-11-16T14:32:31,669 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43457/hbase/data/default/testReplayEditsWrittenViaHRegion/4f290312a7822e6e1180a0e3c454fd9f/.tmp/b/613a3c8ddbf846f6861dac177a93b221 as hdfs://localhost:43457/hbase/data/default/testReplayEditsWrittenViaHRegion/4f290312a7822e6e1180a0e3c454fd9f/b/613a3c8ddbf846f6861dac177a93b221 2024-11-16T14:32:31,675 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43457/hbase/data/default/testReplayEditsWrittenViaHRegion/4f290312a7822e6e1180a0e3c454fd9f/b/613a3c8ddbf846f6861dac177a93b221, entries=10, sequenceid=32, filesize=5.8 K 2024-11-16T14:32:31,676 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~870 B/870, heapSize ~1.80 KB/1840, currentSize=0 B/0 for 4f290312a7822e6e1180a0e3c454fd9f in 33ms, sequenceid=32, compaction requested=false; wal=null 2024-11-16T14:32:31,676 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:43457/hbase/data/default/testReplayEditsWrittenViaHRegion/4f290312a7822e6e1180a0e3c454fd9f/recovered.edits/0000000000000000032 2024-11-16T14:32:31,678 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 4f290312a7822e6e1180a0e3c454fd9f 2024-11-16T14:32:31,678 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 4f290312a7822e6e1180a0e3c454fd9f 2024-11-16T14:32:31,679 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-16T14:32:31,680 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 4f290312a7822e6e1180a0e3c454fd9f 2024-11-16T14:32:31,683 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43457/hbase/data/default/testReplayEditsWrittenViaHRegion/4f290312a7822e6e1180a0e3c454fd9f/recovered.edits/33.seqid, newMaxSeqId=33, maxSeqId=1 2024-11-16T14:32:31,684 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 4f290312a7822e6e1180a0e3c454fd9f; next sequenceid=34; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75339412, jitterRate=0.12264472246170044}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-16T14:32:31,684 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 4f290312a7822e6e1180a0e3c454fd9f: Writing region info on filesystem at 1731767551616Initializing all the Stores at 1731767551617 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731767551617Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731767551618 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731767551618Obtaining lock to block concurrent updates at 1731767551642 (+24 ms)Preparing flush snapshotting stores in 4f290312a7822e6e1180a0e3c454fd9f at 1731767551642Finished memstore snapshotting testReplayEditsWrittenViaHRegion,,1731767551200.4f290312a7822e6e1180a0e3c454fd9f., syncing WAL and waiting on mvcc, flushsize=dataSize=870, getHeapSize=2320, getOffHeapSize=0, getCellsCount=10 at 1731767551642Flushing stores of testReplayEditsWrittenViaHRegion,,1731767551200.4f290312a7822e6e1180a0e3c454fd9f. at 1731767551642Flushing 4f290312a7822e6e1180a0e3c454fd9f/b: creating writer at 1731767551642Flushing 4f290312a7822e6e1180a0e3c454fd9f/b: appending metadata at 1731767551656 (+14 ms)Flushing 4f290312a7822e6e1180a0e3c454fd9f/b: closing flushed file at 1731767551656Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5cb1784f: reopening flushed file at 1731767551669 (+13 ms)Finished flush of dataSize ~870 B/870, heapSize ~1.80 KB/1840, currentSize=0 B/0 for 4f290312a7822e6e1180a0e3c454fd9f in 33ms, sequenceid=32, compaction requested=false; wal=null at 1731767551676 (+7 ms)Cleaning up temporary data from old regions at 1731767551678 (+2 ms)Region opened successfully at 1731767551684 (+6 ms) 2024-11-16T14:32:31,707 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayValueCompression#testReplayEditsAfterPartialFlush Thread=419 (was 410) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1995475656_22 at /127.0.0.1:59894 [Waiting for operation #27] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1995475656_22 at /127.0.0.1:57558 [Waiting for operation #50] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-865919460-172.17.0.2-1731767527819:blk_1073741899_1077, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1995475656_22 at /127.0.0.1:60038 [Receiving block BP-865919460-172.17.0.2-1731767527819:blk_1073741899_1077] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1995475656_22 at /127.0.0.1:37744 [Receiving block BP-865919460-172.17.0.2-1731767527819:blk_1073741899_1077] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-865919460-172.17.0.2-1731767527819:blk_1073741899_1077, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-20-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1995475656_22 at /127.0.0.1:37584 [Waiting for operation #25] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1995475656_22 at /127.0.0.1:57700 [Receiving block BP-865919460-172.17.0.2-1731767527819:blk_1073741899_1077] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-20-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-865919460-172.17.0.2-1731767527819:blk_1073741899_1077, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-20-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1127 (was 1051) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=168 (was 168), ProcessCount=11 (was 11), AvailableMemoryMB=4499 (was 4513) 2024-11-16T14:32:31,708 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1127 is superior to 1024 2024-11-16T14:32:31,720 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayValueCompression#testReplayEditsAfterAbortingFlush Thread=419, OpenFileDescriptor=1127, MaxFileDescriptor=1048576, SystemLoadAverage=168, ProcessCount=11, AvailableMemoryMB=4498 2024-11-16T14:32:31,720 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1127 is superior to 1024 2024-11-16T14:32:31,734 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-16T14:32:31,736 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-16T14:32:31,737 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-16T14:32:31,739 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-54164990, suffix=, logDir=hdfs://localhost:43457/hbase/WALs/hregion-54164990, archiveDir=hdfs://localhost:43457/hbase/oldWALs, maxLogs=32 2024-11-16T14:32:31,751 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-54164990/hregion-54164990.1731767551740, exclude list is [], retry=0 2024-11-16T14:32:31,754 DEBUG [AsyncFSWAL-22-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43279,DS-17ab9554-4509-49b3-918b-178692b9a473,DISK] 2024-11-16T14:32:31,754 DEBUG [AsyncFSWAL-22-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33005,DS-6a1dd528-c0be-4580-a53e-7113879f7f97,DISK] 2024-11-16T14:32:31,756 DEBUG [AsyncFSWAL-22-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36193,DS-a30e140c-8532-4096-8cae-c8d713a3d4e3,DISK] 2024-11-16T14:32:31,758 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-54164990/hregion-54164990.1731767551740 2024-11-16T14:32:31,758 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36015:36015),(127.0.0.1/127.0.0.1:41179:41179),(127.0.0.1/127.0.0.1:35677:35677)] 2024-11-16T14:32:31,759 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => ae16d8c76417ff0160b34b52ec21a96e, NAME => 'testReplayEditsAfterAbortingFlush,,1731767551735.ae16d8c76417ff0160b34b52ec21a96e.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsAfterAbortingFlush', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43457/hbase 2024-11-16T14:32:31,767 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33005 is added to blk_1073741902_1080 (size=68) 2024-11-16T14:32:31,767 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36193 is added to blk_1073741902_1080 (size=68) 2024-11-16T14:32:31,767 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43279 is added to blk_1073741902_1080 (size=68) 2024-11-16T14:32:31,768 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterAbortingFlush,,1731767551735.ae16d8c76417ff0160b34b52ec21a96e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T14:32:31,771 INFO [StoreOpener-ae16d8c76417ff0160b34b52ec21a96e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region ae16d8c76417ff0160b34b52ec21a96e 2024-11-16T14:32:31,772 INFO [StoreOpener-ae16d8c76417ff0160b34b52ec21a96e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ae16d8c76417ff0160b34b52ec21a96e columnFamilyName a 2024-11-16T14:32:31,772 DEBUG [StoreOpener-ae16d8c76417ff0160b34b52ec21a96e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:32:31,773 INFO [StoreOpener-ae16d8c76417ff0160b34b52ec21a96e-1 {}] regionserver.HStore(327): Store=ae16d8c76417ff0160b34b52ec21a96e/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T14:32:31,773 INFO [StoreOpener-ae16d8c76417ff0160b34b52ec21a96e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region ae16d8c76417ff0160b34b52ec21a96e 2024-11-16T14:32:31,774 INFO [StoreOpener-ae16d8c76417ff0160b34b52ec21a96e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ae16d8c76417ff0160b34b52ec21a96e columnFamilyName b 2024-11-16T14:32:31,774 DEBUG [StoreOpener-ae16d8c76417ff0160b34b52ec21a96e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:32:31,775 INFO [StoreOpener-ae16d8c76417ff0160b34b52ec21a96e-1 {}] regionserver.HStore(327): Store=ae16d8c76417ff0160b34b52ec21a96e/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T14:32:31,775 INFO [StoreOpener-ae16d8c76417ff0160b34b52ec21a96e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region ae16d8c76417ff0160b34b52ec21a96e 2024-11-16T14:32:31,776 INFO [StoreOpener-ae16d8c76417ff0160b34b52ec21a96e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ae16d8c76417ff0160b34b52ec21a96e columnFamilyName c 2024-11-16T14:32:31,776 DEBUG [StoreOpener-ae16d8c76417ff0160b34b52ec21a96e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:32:31,777 INFO [StoreOpener-ae16d8c76417ff0160b34b52ec21a96e-1 {}] regionserver.HStore(327): Store=ae16d8c76417ff0160b34b52ec21a96e/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T14:32:31,777 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for ae16d8c76417ff0160b34b52ec21a96e 2024-11-16T14:32:31,778 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43457/hbase/data/default/testReplayEditsAfterAbortingFlush/ae16d8c76417ff0160b34b52ec21a96e 2024-11-16T14:32:31,778 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43457/hbase/data/default/testReplayEditsAfterAbortingFlush/ae16d8c76417ff0160b34b52ec21a96e 2024-11-16T14:32:31,779 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for ae16d8c76417ff0160b34b52ec21a96e 2024-11-16T14:32:31,779 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for ae16d8c76417ff0160b34b52ec21a96e 2024-11-16T14:32:31,779 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterAbortingFlush descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-16T14:32:31,781 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for ae16d8c76417ff0160b34b52ec21a96e 2024-11-16T14:32:31,783 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43457/hbase/data/default/testReplayEditsAfterAbortingFlush/ae16d8c76417ff0160b34b52ec21a96e/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-16T14:32:31,783 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened ae16d8c76417ff0160b34b52ec21a96e; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67108147, jitterRate=-1.068413257598877E-5}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-16T14:32:31,784 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for ae16d8c76417ff0160b34b52ec21a96e: Writing region info on filesystem at 1731767551768Initializing all the Stores at 1731767551769 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731767551769Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731767551771 (+2 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731767551771Cleaning up temporary data from old regions at 1731767551779 (+8 ms)Region opened successfully at 1731767551784 (+5 ms) 2024-11-16T14:32:31,784 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing ae16d8c76417ff0160b34b52ec21a96e, disabling compactions & flushes 2024-11-16T14:32:31,784 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterAbortingFlush,,1731767551735.ae16d8c76417ff0160b34b52ec21a96e. 2024-11-16T14:32:31,784 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterAbortingFlush,,1731767551735.ae16d8c76417ff0160b34b52ec21a96e. 2024-11-16T14:32:31,784 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterAbortingFlush,,1731767551735.ae16d8c76417ff0160b34b52ec21a96e. after waiting 0 ms 2024-11-16T14:32:31,784 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterAbortingFlush,,1731767551735.ae16d8c76417ff0160b34b52ec21a96e. 2024-11-16T14:32:31,785 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsAfterAbortingFlush,,1731767551735.ae16d8c76417ff0160b34b52ec21a96e. 2024-11-16T14:32:31,785 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for ae16d8c76417ff0160b34b52ec21a96e: Waiting for close lock at 1731767551784Disabling compacts and flushes for region at 1731767551784Disabling writes for close at 1731767551784Writing region close event to WAL at 1731767551785 (+1 ms)Closed at 1731767551785 2024-11-16T14:32:31,788 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /hbase/WALs/hregion-54164990/hregion-54164990.1731767551740 not finished, retry = 0 2024-11-16T14:32:31,788 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43279 is added to blk_1073741901_1079 (size=95) 2024-11-16T14:32:31,788 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33005 is added to blk_1073741901_1079 (size=95) 2024-11-16T14:32:31,788 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36193 is added to blk_1073741901_1079 (size=95) 2024-11-16T14:32:31,894 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-16T14:32:31,894 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-54164990:(num 1731767551740) 2024-11-16T14:32:31,895 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-16T14:32:31,899 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:43457/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731767551733, archiveDir=hdfs://localhost:43457/hbase/oldWALs, maxLogs=32 2024-11-16T14:32:31,911 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731767551733/wal.1731767551900, exclude list is [], retry=0 2024-11-16T14:32:31,914 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43279,DS-17ab9554-4509-49b3-918b-178692b9a473,DISK] 2024-11-16T14:32:31,914 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33005,DS-6a1dd528-c0be-4580-a53e-7113879f7f97,DISK] 2024-11-16T14:32:31,914 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36193,DS-a30e140c-8532-4096-8cae-c8d713a3d4e3,DISK] 2024-11-16T14:32:31,916 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731767551733/wal.1731767551900 2024-11-16T14:32:31,916 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36015:36015),(127.0.0.1/127.0.0.1:41179:41179),(127.0.0.1/127.0.0.1:35677:35677)] 2024-11-16T14:32:31,982 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => ae16d8c76417ff0160b34b52ec21a96e, NAME => 'testReplayEditsAfterAbortingFlush,,1731767551735.ae16d8c76417ff0160b34b52ec21a96e.', STARTKEY => '', ENDKEY => ''} 2024-11-16T14:32:31,984 DEBUG [Time-limited test {}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterAbortingFlush ae16d8c76417ff0160b34b52ec21a96e 2024-11-16T14:32:31,984 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterAbortingFlush,,1731767551735.ae16d8c76417ff0160b34b52ec21a96e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T14:32:31,984 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for ae16d8c76417ff0160b34b52ec21a96e 2024-11-16T14:32:31,984 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for ae16d8c76417ff0160b34b52ec21a96e 2024-11-16T14:32:31,986 INFO [StoreOpener-ae16d8c76417ff0160b34b52ec21a96e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region ae16d8c76417ff0160b34b52ec21a96e 2024-11-16T14:32:31,987 INFO [StoreOpener-ae16d8c76417ff0160b34b52ec21a96e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ae16d8c76417ff0160b34b52ec21a96e columnFamilyName a 2024-11-16T14:32:31,988 DEBUG [StoreOpener-ae16d8c76417ff0160b34b52ec21a96e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:32:31,988 INFO [StoreOpener-ae16d8c76417ff0160b34b52ec21a96e-1 {}] regionserver.HStore(327): Store=ae16d8c76417ff0160b34b52ec21a96e/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T14:32:31,989 INFO [StoreOpener-ae16d8c76417ff0160b34b52ec21a96e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region ae16d8c76417ff0160b34b52ec21a96e 2024-11-16T14:32:31,990 INFO [StoreOpener-ae16d8c76417ff0160b34b52ec21a96e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ae16d8c76417ff0160b34b52ec21a96e columnFamilyName b 2024-11-16T14:32:31,990 DEBUG [StoreOpener-ae16d8c76417ff0160b34b52ec21a96e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:32:31,990 INFO [StoreOpener-ae16d8c76417ff0160b34b52ec21a96e-1 {}] regionserver.HStore(327): Store=ae16d8c76417ff0160b34b52ec21a96e/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T14:32:31,990 INFO [StoreOpener-ae16d8c76417ff0160b34b52ec21a96e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region ae16d8c76417ff0160b34b52ec21a96e 2024-11-16T14:32:31,991 INFO [StoreOpener-ae16d8c76417ff0160b34b52ec21a96e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ae16d8c76417ff0160b34b52ec21a96e columnFamilyName c 2024-11-16T14:32:31,991 DEBUG [StoreOpener-ae16d8c76417ff0160b34b52ec21a96e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:32:31,992 INFO [StoreOpener-ae16d8c76417ff0160b34b52ec21a96e-1 {}] regionserver.HStore(327): Store=ae16d8c76417ff0160b34b52ec21a96e/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T14:32:31,992 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for ae16d8c76417ff0160b34b52ec21a96e 2024-11-16T14:32:31,993 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43457/hbase/data/default/testReplayEditsAfterAbortingFlush/ae16d8c76417ff0160b34b52ec21a96e 2024-11-16T14:32:31,994 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43457/hbase/data/default/testReplayEditsAfterAbortingFlush/ae16d8c76417ff0160b34b52ec21a96e 2024-11-16T14:32:31,995 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for ae16d8c76417ff0160b34b52ec21a96e 2024-11-16T14:32:31,995 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for ae16d8c76417ff0160b34b52ec21a96e 2024-11-16T14:32:31,996 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterAbortingFlush descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-16T14:32:31,997 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for ae16d8c76417ff0160b34b52ec21a96e 2024-11-16T14:32:31,998 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened ae16d8c76417ff0160b34b52ec21a96e; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72242544, jitterRate=0.07649779319763184}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-16T14:32:31,998 DEBUG [Time-limited test {}] regionserver.HRegion(1122): Running coprocessor post-open hooks for ae16d8c76417ff0160b34b52ec21a96e 2024-11-16T14:32:31,998 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for ae16d8c76417ff0160b34b52ec21a96e: Running coprocessor pre-open hook at 1731767551985Writing region info on filesystem at 1731767551985Initializing all the Stores at 1731767551986 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731767551986Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731767551986Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731767551986Cleaning up temporary data from old regions at 1731767551995 (+9 ms)Running coprocessor post-open hooks at 1731767551998 (+3 ms)Region opened successfully at 1731767551998 2024-11-16T14:32:32,010 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing ae16d8c76417ff0160b34b52ec21a96e 3/3 column families, dataSize=590 B heapSize=2.08 KB 2024-11-16T14:32:32,011 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for ae16d8c76417ff0160b34b52ec21a96e/a, retrying num=0 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:32:32,430 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenViaHRegion 2024-11-16T14:32:32,430 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenViaHRegion Metrics about Tables on a single HBase RegionServer 2024-11-16T14:32:32,431 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testSequentialEditLogSeqNum 2024-11-16T14:32:32,431 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testSequentialEditLogSeqNum Metrics about Tables on a single HBase RegionServer 2024-11-16T14:32:32,432 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterRegionMovedWithMultiCF 2024-11-16T14:32:32,432 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterRegionMovedWithMultiCF Metrics about Tables on a single HBase RegionServer 2024-11-16T14:32:32,432 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRegionMadeOfBulkLoadedFilesOnly 2024-11-16T14:32:32,432 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testRegionMadeOfBulkLoadedFilesOnly Metrics about Tables on a single HBase RegionServer 2024-11-16T14:32:32,433 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterAbortingFlush 2024-11-16T14:32:32,433 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterAbortingFlush Metrics about Tables on a single HBase RegionServer 2024-11-16T14:32:33,012 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for ae16d8c76417ff0160b34b52ec21a96e/a, retrying num=1 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:32:33,677 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-16T14:32:34,013 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for ae16d8c76417ff0160b34b52ec21a96e/a, retrying num=2 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:32:35,015 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for ae16d8c76417ff0160b34b52ec21a96e/a, retrying num=3 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:32:36,016 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for ae16d8c76417ff0160b34b52ec21a96e/a, retrying num=4 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:32:37,017 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for ae16d8c76417ff0160b34b52ec21a96e/a, retrying num=5 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:32:38,018 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for ae16d8c76417ff0160b34b52ec21a96e/a, retrying num=6 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:32:39,020 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for ae16d8c76417ff0160b34b52ec21a96e/a, retrying num=7 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:32:40,021 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for ae16d8c76417ff0160b34b52ec21a96e/a, retrying num=8 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:32:41,022 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for ae16d8c76417ff0160b34b52ec21a96e/a, retrying num=9 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:32:41,025 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for ae16d8c76417ff0160b34b52ec21a96e: 2024-11-16T14:32:41,025 INFO [Time-limited test {}] wal.AbstractTestWALReplay(671): Expected simulated exception when flushing region, region: testReplayEditsAfterAbortingFlush,,1731767551735.ae16d8c76417ff0160b34b52ec21a96e. 2024-11-16T14:32:41,043 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for ae16d8c76417ff0160b34b52ec21a96e: 2024-11-16T14:32:41,043 INFO [Time-limited test {}] wal.AbstractTestWALReplay(691): Expected exception when flushing region because server is stopped,Aborting flush because server is aborted... 2024-11-16T14:32:41,043 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing ae16d8c76417ff0160b34b52ec21a96e, disabling compactions & flushes 2024-11-16T14:32:41,043 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterAbortingFlush,,1731767551735.ae16d8c76417ff0160b34b52ec21a96e. 2024-11-16T14:32:41,043 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterAbortingFlush,,1731767551735.ae16d8c76417ff0160b34b52ec21a96e. 2024-11-16T14:32:41,043 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterAbortingFlush,,1731767551735.ae16d8c76417ff0160b34b52ec21a96e. after waiting 0 ms 2024-11-16T14:32:41,043 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterAbortingFlush,,1731767551735.ae16d8c76417ff0160b34b52ec21a96e. 2024-11-16T14:32:41,044 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 1190 in region testReplayEditsAfterAbortingFlush,,1731767551735.ae16d8c76417ff0160b34b52ec21a96e. 2024-11-16T14:32:41,044 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsAfterAbortingFlush,,1731767551735.ae16d8c76417ff0160b34b52ec21a96e. 2024-11-16T14:32:41,044 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for ae16d8c76417ff0160b34b52ec21a96e: Waiting for close lock at 1731767561043Running coprocessor pre-close hooks at 1731767561043Disabling compacts and flushes for region at 1731767561043Disabling writes for close at 1731767561043Writing region close event to WAL at 1731767561044 (+1 ms)Running coprocessor post-close hooks at 1731767561044Closed at 1731767561044 2024-11-16T14:32:41,046 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36193 is added to blk_1073741903_1081 (size=2685) 2024-11-16T14:32:41,046 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33005 is added to blk_1073741903_1081 (size=2685) 2024-11-16T14:32:41,047 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43279 is added to blk_1073741903_1081 (size=2685) 2024-11-16T14:32:41,062 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:43457/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731767551733/wal.1731767551900, size=2.6 K (2685bytes) 2024-11-16T14:32:41,062 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:43457/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731767551733/wal.1731767551900 2024-11-16T14:32:41,062 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:43457/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731767551733/wal.1731767551900 after 0ms 2024-11-16T14:32:41,064 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:43457/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731767551733/wal.1731767551900: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-16T14:32:41,064 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:43457/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731767551733/wal.1731767551900 took 2ms 2024-11-16T14:32:41,066 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:43457/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731767551733/wal.1731767551900 so closing down 2024-11-16T14:32:41,067 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-16T14:32:41,067 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-wal.1731767551900.temp 2024-11-16T14:32:41,068 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:43457/hbase/data/default/testReplayEditsAfterAbortingFlush/ae16d8c76417ff0160b34b52ec21a96e/recovered.edits/0000000000000000004-wal.1731767551900.temp 2024-11-16T14:32:41,069 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-16T14:32:41,075 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36193 is added to blk_1073741904_1082 (size=2094) 2024-11-16T14:32:41,076 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33005 is added to blk_1073741904_1082 (size=2094) 2024-11-16T14:32:41,076 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43279 is added to blk_1073741904_1082 (size=2094) 2024-11-16T14:32:41,076 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:43457/hbase/data/default/testReplayEditsAfterAbortingFlush/ae16d8c76417ff0160b34b52ec21a96e/recovered.edits/0000000000000000004-wal.1731767551900.temp (wrote 20 edits, skipped 0 edits in 0 ms) 2024-11-16T14:32:41,078 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:43457/hbase/data/default/testReplayEditsAfterAbortingFlush/ae16d8c76417ff0160b34b52ec21a96e/recovered.edits/0000000000000000004-wal.1731767551900.temp to hdfs://localhost:43457/hbase/data/default/testReplayEditsAfterAbortingFlush/ae16d8c76417ff0160b34b52ec21a96e/recovered.edits/0000000000000000026 2024-11-16T14:32:41,078 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 23 edits across 1 Regions in 13 ms; skipped=3; WAL=hdfs://localhost:43457/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731767551733/wal.1731767551900, size=2.6 K, length=2685, corrupted=false, cancelled=false 2024-11-16T14:32:41,078 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:43457/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731767551733/wal.1731767551900, journal: Splitting hdfs://localhost:43457/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731767551733/wal.1731767551900, size=2.6 K (2685bytes) at 1731767561062Finishing writing output for hdfs://localhost:43457/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731767551733/wal.1731767551900 so closing down at 1731767561066 (+4 ms)Creating recovered edits writer path=hdfs://localhost:43457/hbase/data/default/testReplayEditsAfterAbortingFlush/ae16d8c76417ff0160b34b52ec21a96e/recovered.edits/0000000000000000004-wal.1731767551900.temp at 1731767561069 (+3 ms)3 split writer threads finished at 1731767561069Closed recovered edits writer path=hdfs://localhost:43457/hbase/data/default/testReplayEditsAfterAbortingFlush/ae16d8c76417ff0160b34b52ec21a96e/recovered.edits/0000000000000000004-wal.1731767551900.temp (wrote 20 edits, skipped 0 edits in 0 ms) at 1731767561076 (+7 ms)Rename recovered edits hdfs://localhost:43457/hbase/data/default/testReplayEditsAfterAbortingFlush/ae16d8c76417ff0160b34b52ec21a96e/recovered.edits/0000000000000000004-wal.1731767551900.temp to hdfs://localhost:43457/hbase/data/default/testReplayEditsAfterAbortingFlush/ae16d8c76417ff0160b34b52ec21a96e/recovered.edits/0000000000000000026 at 1731767561078 (+2 ms)Processed 23 edits across 1 Regions in 13 ms; skipped=3; WAL=hdfs://localhost:43457/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731767551733/wal.1731767551900, size=2.6 K, length=2685, corrupted=false, cancelled=false at 1731767561078 2024-11-16T14:32:41,080 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:43457/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731767551733/wal.1731767551900 to hdfs://localhost:43457/hbase/oldWALs/wal.1731767551900 2024-11-16T14:32:41,080 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:43457/hbase/data/default/testReplayEditsAfterAbortingFlush/ae16d8c76417ff0160b34b52ec21a96e/recovered.edits/0000000000000000026 2024-11-16T14:32:41,080 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-16T14:32:41,082 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:43457/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731767551733, archiveDir=hdfs://localhost:43457/hbase/oldWALs, maxLogs=32 2024-11-16T14:32:41,094 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731767551733/wal.1731767561082, exclude list is [], retry=0 2024-11-16T14:32:41,096 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43279,DS-17ab9554-4509-49b3-918b-178692b9a473,DISK] 2024-11-16T14:32:41,097 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33005,DS-6a1dd528-c0be-4580-a53e-7113879f7f97,DISK] 2024-11-16T14:32:41,097 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36193,DS-a30e140c-8532-4096-8cae-c8d713a3d4e3,DISK] 2024-11-16T14:32:41,098 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731767551733/wal.1731767561082 2024-11-16T14:32:41,099 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36015:36015),(127.0.0.1/127.0.0.1:41179:41179),(127.0.0.1/127.0.0.1:35677:35677)] 2024-11-16T14:32:41,099 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => ae16d8c76417ff0160b34b52ec21a96e, NAME => 'testReplayEditsAfterAbortingFlush,,1731767551735.ae16d8c76417ff0160b34b52ec21a96e.', STARTKEY => '', ENDKEY => ''} 2024-11-16T14:32:41,099 DEBUG [Time-limited test {}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterAbortingFlush ae16d8c76417ff0160b34b52ec21a96e 2024-11-16T14:32:41,099 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterAbortingFlush,,1731767551735.ae16d8c76417ff0160b34b52ec21a96e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T14:32:41,100 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for ae16d8c76417ff0160b34b52ec21a96e 2024-11-16T14:32:41,100 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for ae16d8c76417ff0160b34b52ec21a96e 2024-11-16T14:32:41,101 INFO [StoreOpener-ae16d8c76417ff0160b34b52ec21a96e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region ae16d8c76417ff0160b34b52ec21a96e 2024-11-16T14:32:41,102 INFO [StoreOpener-ae16d8c76417ff0160b34b52ec21a96e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ae16d8c76417ff0160b34b52ec21a96e columnFamilyName a 2024-11-16T14:32:41,103 DEBUG [StoreOpener-ae16d8c76417ff0160b34b52ec21a96e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:32:41,103 INFO [StoreOpener-ae16d8c76417ff0160b34b52ec21a96e-1 {}] regionserver.HStore(327): Store=ae16d8c76417ff0160b34b52ec21a96e/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T14:32:41,103 INFO [StoreOpener-ae16d8c76417ff0160b34b52ec21a96e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region ae16d8c76417ff0160b34b52ec21a96e 2024-11-16T14:32:41,104 INFO [StoreOpener-ae16d8c76417ff0160b34b52ec21a96e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ae16d8c76417ff0160b34b52ec21a96e columnFamilyName b 2024-11-16T14:32:41,104 DEBUG [StoreOpener-ae16d8c76417ff0160b34b52ec21a96e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:32:41,105 INFO [StoreOpener-ae16d8c76417ff0160b34b52ec21a96e-1 {}] regionserver.HStore(327): Store=ae16d8c76417ff0160b34b52ec21a96e/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T14:32:41,105 INFO [StoreOpener-ae16d8c76417ff0160b34b52ec21a96e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region ae16d8c76417ff0160b34b52ec21a96e 2024-11-16T14:32:41,106 INFO [StoreOpener-ae16d8c76417ff0160b34b52ec21a96e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ae16d8c76417ff0160b34b52ec21a96e columnFamilyName c 2024-11-16T14:32:41,106 DEBUG [StoreOpener-ae16d8c76417ff0160b34b52ec21a96e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:32:41,106 INFO [StoreOpener-ae16d8c76417ff0160b34b52ec21a96e-1 {}] regionserver.HStore(327): Store=ae16d8c76417ff0160b34b52ec21a96e/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T14:32:41,106 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for ae16d8c76417ff0160b34b52ec21a96e 2024-11-16T14:32:41,107 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43457/hbase/data/default/testReplayEditsAfterAbortingFlush/ae16d8c76417ff0160b34b52ec21a96e 2024-11-16T14:32:41,108 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:43457/hbase/data/default/testReplayEditsAfterAbortingFlush/ae16d8c76417ff0160b34b52ec21a96e 2024-11-16T14:32:41,108 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:43457/hbase/data/default/testReplayEditsAfterAbortingFlush/ae16d8c76417ff0160b34b52ec21a96e/recovered.edits/0000000000000000026 2024-11-16T14:32:41,110 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:43457/hbase/data/default/testReplayEditsAfterAbortingFlush/ae16d8c76417ff0160b34b52ec21a96e/recovered.edits/0000000000000000026: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-16T14:32:41,112 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 20, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=26, path=hdfs://localhost:43457/hbase/data/default/testReplayEditsAfterAbortingFlush/ae16d8c76417ff0160b34b52ec21a96e/recovered.edits/0000000000000000026 2024-11-16T14:32:41,112 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing ae16d8c76417ff0160b34b52ec21a96e 3/3 column families, dataSize=1.16 KB heapSize=3.41 KB 2024-11-16T14:32:41,126 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43457/hbase/data/default/testReplayEditsAfterAbortingFlush/ae16d8c76417ff0160b34b52ec21a96e/.tmp/a/86c3a2174b584317a6236ac7e2f7b7ac is 64, key is testReplayEditsAfterAbortingFlush12/a:q/1731767561035/Put/seqid=0 2024-11-16T14:32:41,132 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36193 is added to blk_1073741906_1084 (size=5523) 2024-11-16T14:32:41,132 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33005 is added to blk_1073741906_1084 (size=5523) 2024-11-16T14:32:41,133 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43279 is added to blk_1073741906_1084 (size=5523) 2024-11-16T14:32:41,133 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=416 B at sequenceid=26 (bloomFilter=true), to=hdfs://localhost:43457/hbase/data/default/testReplayEditsAfterAbortingFlush/ae16d8c76417ff0160b34b52ec21a96e/.tmp/a/86c3a2174b584317a6236ac7e2f7b7ac 2024-11-16T14:32:41,151 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43457/hbase/data/default/testReplayEditsAfterAbortingFlush/ae16d8c76417ff0160b34b52ec21a96e/.tmp/b/63e43c0685de4e12b2a2723970dffa0a is 64, key is testReplayEditsAfterAbortingFlush10/b:q/1731767561026/Put/seqid=0 2024-11-16T14:32:41,158 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36193 is added to blk_1073741907_1085 (size=5524) 2024-11-16T14:32:41,158 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33005 is added to blk_1073741907_1085 (size=5524) 2024-11-16T14:32:41,158 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43279 is added to blk_1073741907_1085 (size=5524) 2024-11-16T14:32:41,159 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=417 B at sequenceid=26 (bloomFilter=true), to=hdfs://localhost:43457/hbase/data/default/testReplayEditsAfterAbortingFlush/ae16d8c76417ff0160b34b52ec21a96e/.tmp/b/63e43c0685de4e12b2a2723970dffa0a 2024-11-16T14:32:41,178 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43457/hbase/data/default/testReplayEditsAfterAbortingFlush/ae16d8c76417ff0160b34b52ec21a96e/.tmp/c/c0268ea1a57242c39260a5ab94887366 is 64, key is testReplayEditsAfterAbortingFlush11/c:q/1731767561032/Put/seqid=0 2024-11-16T14:32:41,184 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36193 is added to blk_1073741908_1086 (size=5457) 2024-11-16T14:32:41,184 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43279 is added to blk_1073741908_1086 (size=5457) 2024-11-16T14:32:41,184 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33005 is added to blk_1073741908_1086 (size=5457) 2024-11-16T14:32:41,185 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=357 B at sequenceid=26 (bloomFilter=true), to=hdfs://localhost:43457/hbase/data/default/testReplayEditsAfterAbortingFlush/ae16d8c76417ff0160b34b52ec21a96e/.tmp/c/c0268ea1a57242c39260a5ab94887366 2024-11-16T14:32:41,192 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43457/hbase/data/default/testReplayEditsAfterAbortingFlush/ae16d8c76417ff0160b34b52ec21a96e/.tmp/a/86c3a2174b584317a6236ac7e2f7b7ac as hdfs://localhost:43457/hbase/data/default/testReplayEditsAfterAbortingFlush/ae16d8c76417ff0160b34b52ec21a96e/a/86c3a2174b584317a6236ac7e2f7b7ac 2024-11-16T14:32:41,197 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43457/hbase/data/default/testReplayEditsAfterAbortingFlush/ae16d8c76417ff0160b34b52ec21a96e/a/86c3a2174b584317a6236ac7e2f7b7ac, entries=7, sequenceid=26, filesize=5.4 K 2024-11-16T14:32:41,198 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43457/hbase/data/default/testReplayEditsAfterAbortingFlush/ae16d8c76417ff0160b34b52ec21a96e/.tmp/b/63e43c0685de4e12b2a2723970dffa0a as hdfs://localhost:43457/hbase/data/default/testReplayEditsAfterAbortingFlush/ae16d8c76417ff0160b34b52ec21a96e/b/63e43c0685de4e12b2a2723970dffa0a 2024-11-16T14:32:41,203 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43457/hbase/data/default/testReplayEditsAfterAbortingFlush/ae16d8c76417ff0160b34b52ec21a96e/b/63e43c0685de4e12b2a2723970dffa0a, entries=7, sequenceid=26, filesize=5.4 K 2024-11-16T14:32:41,205 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43457/hbase/data/default/testReplayEditsAfterAbortingFlush/ae16d8c76417ff0160b34b52ec21a96e/.tmp/c/c0268ea1a57242c39260a5ab94887366 as hdfs://localhost:43457/hbase/data/default/testReplayEditsAfterAbortingFlush/ae16d8c76417ff0160b34b52ec21a96e/c/c0268ea1a57242c39260a5ab94887366 2024-11-16T14:32:41,210 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43457/hbase/data/default/testReplayEditsAfterAbortingFlush/ae16d8c76417ff0160b34b52ec21a96e/c/c0268ea1a57242c39260a5ab94887366, entries=6, sequenceid=26, filesize=5.3 K 2024-11-16T14:32:41,210 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~1.16 KB/1190, heapSize ~3.36 KB/3440, currentSize=0 B/0 for ae16d8c76417ff0160b34b52ec21a96e in 98ms, sequenceid=26, compaction requested=false; wal=null 2024-11-16T14:32:41,211 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:43457/hbase/data/default/testReplayEditsAfterAbortingFlush/ae16d8c76417ff0160b34b52ec21a96e/recovered.edits/0000000000000000026 2024-11-16T14:32:41,212 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for ae16d8c76417ff0160b34b52ec21a96e 2024-11-16T14:32:41,212 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for ae16d8c76417ff0160b34b52ec21a96e 2024-11-16T14:32:41,213 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterAbortingFlush descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-16T14:32:41,215 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for ae16d8c76417ff0160b34b52ec21a96e 2024-11-16T14:32:41,217 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43457/hbase/data/default/testReplayEditsAfterAbortingFlush/ae16d8c76417ff0160b34b52ec21a96e/recovered.edits/26.seqid, newMaxSeqId=26, maxSeqId=1 2024-11-16T14:32:41,218 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened ae16d8c76417ff0160b34b52ec21a96e; next sequenceid=27; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62269216, jitterRate=-0.07211637496948242}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-16T14:32:41,218 DEBUG [Time-limited test {}] regionserver.HRegion(1122): Running coprocessor post-open hooks for ae16d8c76417ff0160b34b52ec21a96e 2024-11-16T14:32:41,219 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for ae16d8c76417ff0160b34b52ec21a96e: Running coprocessor pre-open hook at 1731767561100Writing region info on filesystem at 1731767561100Initializing all the Stores at 1731767561101 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731767561101Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731767561101Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731767561101Obtaining lock to block concurrent updates at 1731767561112 (+11 ms)Preparing flush snapshotting stores in ae16d8c76417ff0160b34b52ec21a96e at 1731767561112Finished memstore snapshotting testReplayEditsAfterAbortingFlush,,1731767551735.ae16d8c76417ff0160b34b52ec21a96e., syncing WAL and waiting on mvcc, flushsize=dataSize=1190, getHeapSize=3440, getOffHeapSize=0, getCellsCount=20 at 1731767561112Flushing stores of testReplayEditsAfterAbortingFlush,,1731767551735.ae16d8c76417ff0160b34b52ec21a96e. at 1731767561112Flushing ae16d8c76417ff0160b34b52ec21a96e/a: creating writer at 1731767561112Flushing ae16d8c76417ff0160b34b52ec21a96e/a: appending metadata at 1731767561126 (+14 ms)Flushing ae16d8c76417ff0160b34b52ec21a96e/a: closing flushed file at 1731767561126Flushing ae16d8c76417ff0160b34b52ec21a96e/b: creating writer at 1731767561138 (+12 ms)Flushing ae16d8c76417ff0160b34b52ec21a96e/b: appending metadata at 1731767561151 (+13 ms)Flushing ae16d8c76417ff0160b34b52ec21a96e/b: closing flushed file at 1731767561151Flushing ae16d8c76417ff0160b34b52ec21a96e/c: creating writer at 1731767561164 (+13 ms)Flushing ae16d8c76417ff0160b34b52ec21a96e/c: appending metadata at 1731767561178 (+14 ms)Flushing ae16d8c76417ff0160b34b52ec21a96e/c: closing flushed file at 1731767561178Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@547c3569: reopening flushed file at 1731767561191 (+13 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@27a4a3ff: reopening flushed file at 1731767561197 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2f50ae62: reopening flushed file at 1731767561204 (+7 ms)Finished flush of dataSize ~1.16 KB/1190, heapSize ~3.36 KB/3440, currentSize=0 B/0 for ae16d8c76417ff0160b34b52ec21a96e in 98ms, sequenceid=26, compaction requested=false; wal=null at 1731767561210 (+6 ms)Cleaning up temporary data from old regions at 1731767561212 (+2 ms)Running coprocessor post-open hooks at 1731767561218 (+6 ms)Region opened successfully at 1731767561219 (+1 ms) 2024-11-16T14:32:41,237 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayValueCompression#testReplayEditsAfterAbortingFlush Thread=419 (was 419), OpenFileDescriptor=1183 (was 1127) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=142 (was 168), ProcessCount=11 (was 11), AvailableMemoryMB=4453 (was 4498) 2024-11-16T14:32:41,237 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1183 is superior to 1024 2024-11-16T14:32:41,249 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayValueCompression#testDatalossWhenInputError Thread=419, OpenFileDescriptor=1183, MaxFileDescriptor=1048576, SystemLoadAverage=142, ProcessCount=11, AvailableMemoryMB=4450 2024-11-16T14:32:41,249 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1183 is superior to 1024 2024-11-16T14:32:41,263 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-16T14:32:41,264 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-16T14:32:41,266 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-16T14:32:41,267 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-16T14:32:41,269 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-13912583, suffix=, logDir=hdfs://localhost:43457/hbase/WALs/hregion-13912583, archiveDir=hdfs://localhost:43457/hbase/oldWALs, maxLogs=32 2024-11-16T14:32:41,282 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-13912583/hregion-13912583.1731767561270, exclude list is [], retry=0 2024-11-16T14:32:41,285 DEBUG [AsyncFSWAL-24-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43279,DS-17ab9554-4509-49b3-918b-178692b9a473,DISK] 2024-11-16T14:32:41,285 DEBUG [AsyncFSWAL-24-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33005,DS-6a1dd528-c0be-4580-a53e-7113879f7f97,DISK] 2024-11-16T14:32:41,286 DEBUG [AsyncFSWAL-24-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36193,DS-a30e140c-8532-4096-8cae-c8d713a3d4e3,DISK] 2024-11-16T14:32:41,288 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-13912583/hregion-13912583.1731767561270 2024-11-16T14:32:41,288 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36015:36015),(127.0.0.1/127.0.0.1:41179:41179),(127.0.0.1/127.0.0.1:35677:35677)] 2024-11-16T14:32:41,288 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 5ff5ce691344c1e06fbe9ec1c997f9bf, NAME => 'testDatalossWhenInputError,,1731767561265.5ff5ce691344c1e06fbe9ec1c997f9bf.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testDatalossWhenInputError', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43457/hbase 2024-11-16T14:32:41,301 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33005 is added to blk_1073741910_1088 (size=61) 2024-11-16T14:32:41,301 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36193 is added to blk_1073741910_1088 (size=61) 2024-11-16T14:32:41,302 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43279 is added to blk_1073741910_1088 (size=61) 2024-11-16T14:32:41,302 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1731767561265.5ff5ce691344c1e06fbe9ec1c997f9bf.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T14:32:41,304 INFO [StoreOpener-5ff5ce691344c1e06fbe9ec1c997f9bf-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 5ff5ce691344c1e06fbe9ec1c997f9bf 2024-11-16T14:32:41,306 INFO [StoreOpener-5ff5ce691344c1e06fbe9ec1c997f9bf-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5ff5ce691344c1e06fbe9ec1c997f9bf columnFamilyName a 2024-11-16T14:32:41,306 DEBUG [StoreOpener-5ff5ce691344c1e06fbe9ec1c997f9bf-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:32:41,306 INFO [StoreOpener-5ff5ce691344c1e06fbe9ec1c997f9bf-1 {}] regionserver.HStore(327): Store=5ff5ce691344c1e06fbe9ec1c997f9bf/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T14:32:41,306 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 5ff5ce691344c1e06fbe9ec1c997f9bf 2024-11-16T14:32:41,307 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43457/hbase/data/default/testDatalossWhenInputError/5ff5ce691344c1e06fbe9ec1c997f9bf 2024-11-16T14:32:41,308 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43457/hbase/data/default/testDatalossWhenInputError/5ff5ce691344c1e06fbe9ec1c997f9bf 2024-11-16T14:32:41,308 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 5ff5ce691344c1e06fbe9ec1c997f9bf 2024-11-16T14:32:41,308 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 5ff5ce691344c1e06fbe9ec1c997f9bf 2024-11-16T14:32:41,310 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 5ff5ce691344c1e06fbe9ec1c997f9bf 2024-11-16T14:32:41,312 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43457/hbase/data/default/testDatalossWhenInputError/5ff5ce691344c1e06fbe9ec1c997f9bf/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-16T14:32:41,313 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 5ff5ce691344c1e06fbe9ec1c997f9bf; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67371286, jitterRate=0.00391039252281189}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-16T14:32:41,313 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 5ff5ce691344c1e06fbe9ec1c997f9bf: Writing region info on filesystem at 1731767561302Initializing all the Stores at 1731767561303 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731767561303Cleaning up temporary data from old regions at 1731767561308 (+5 ms)Region opened successfully at 1731767561313 (+5 ms) 2024-11-16T14:32:41,313 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 5ff5ce691344c1e06fbe9ec1c997f9bf, disabling compactions & flushes 2024-11-16T14:32:41,313 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testDatalossWhenInputError,,1731767561265.5ff5ce691344c1e06fbe9ec1c997f9bf. 2024-11-16T14:32:41,313 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testDatalossWhenInputError,,1731767561265.5ff5ce691344c1e06fbe9ec1c997f9bf. 2024-11-16T14:32:41,314 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testDatalossWhenInputError,,1731767561265.5ff5ce691344c1e06fbe9ec1c997f9bf. after waiting 1 ms 2024-11-16T14:32:41,314 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testDatalossWhenInputError,,1731767561265.5ff5ce691344c1e06fbe9ec1c997f9bf. 2024-11-16T14:32:41,314 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testDatalossWhenInputError,,1731767561265.5ff5ce691344c1e06fbe9ec1c997f9bf. 2024-11-16T14:32:41,314 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 5ff5ce691344c1e06fbe9ec1c997f9bf: Waiting for close lock at 1731767561313Disabling compacts and flushes for region at 1731767561313Disabling writes for close at 1731767561314 (+1 ms)Writing region close event to WAL at 1731767561314Closed at 1731767561314 2024-11-16T14:32:41,318 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43279 is added to blk_1073741909_1087 (size=95) 2024-11-16T14:32:41,318 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33005 is added to blk_1073741909_1087 (size=95) 2024-11-16T14:32:41,318 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36193 is added to blk_1073741909_1087 (size=95) 2024-11-16T14:32:41,320 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-16T14:32:41,320 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-13912583:(num 1731767561270) 2024-11-16T14:32:41,320 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-16T14:32:41,323 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:43457/hbase/WALs/testdatalosswheninputerror-manual,16010,1731767561264, archiveDir=hdfs://localhost:43457/hbase/oldWALs, maxLogs=32 2024-11-16T14:32:41,334 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testdatalosswheninputerror-manual,16010,1731767561264/wal.1731767561323, exclude list is [], retry=0 2024-11-16T14:32:41,337 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33005,DS-6a1dd528-c0be-4580-a53e-7113879f7f97,DISK] 2024-11-16T14:32:41,337 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36193,DS-a30e140c-8532-4096-8cae-c8d713a3d4e3,DISK] 2024-11-16T14:32:41,337 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43279,DS-17ab9554-4509-49b3-918b-178692b9a473,DISK] 2024-11-16T14:32:41,338 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testdatalosswheninputerror-manual,16010,1731767561264/wal.1731767561323 2024-11-16T14:32:41,339 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41179:41179),(127.0.0.1/127.0.0.1:35677:35677),(127.0.0.1/127.0.0.1:36015:36015)] 2024-11-16T14:32:41,339 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 5ff5ce691344c1e06fbe9ec1c997f9bf, NAME => 'testDatalossWhenInputError,,1731767561265.5ff5ce691344c1e06fbe9ec1c997f9bf.', STARTKEY => '', ENDKEY => ''} 2024-11-16T14:32:41,339 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1731767561265.5ff5ce691344c1e06fbe9ec1c997f9bf.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T14:32:41,339 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 5ff5ce691344c1e06fbe9ec1c997f9bf 2024-11-16T14:32:41,339 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 5ff5ce691344c1e06fbe9ec1c997f9bf 2024-11-16T14:32:41,340 INFO [StoreOpener-5ff5ce691344c1e06fbe9ec1c997f9bf-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 5ff5ce691344c1e06fbe9ec1c997f9bf 2024-11-16T14:32:41,341 INFO [StoreOpener-5ff5ce691344c1e06fbe9ec1c997f9bf-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5ff5ce691344c1e06fbe9ec1c997f9bf columnFamilyName a 2024-11-16T14:32:41,341 DEBUG [StoreOpener-5ff5ce691344c1e06fbe9ec1c997f9bf-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:32:41,342 INFO [StoreOpener-5ff5ce691344c1e06fbe9ec1c997f9bf-1 {}] regionserver.HStore(327): Store=5ff5ce691344c1e06fbe9ec1c997f9bf/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T14:32:41,342 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 5ff5ce691344c1e06fbe9ec1c997f9bf 2024-11-16T14:32:41,342 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43457/hbase/data/default/testDatalossWhenInputError/5ff5ce691344c1e06fbe9ec1c997f9bf 2024-11-16T14:32:41,343 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43457/hbase/data/default/testDatalossWhenInputError/5ff5ce691344c1e06fbe9ec1c997f9bf 2024-11-16T14:32:41,344 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 5ff5ce691344c1e06fbe9ec1c997f9bf 2024-11-16T14:32:41,344 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 5ff5ce691344c1e06fbe9ec1c997f9bf 2024-11-16T14:32:41,345 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 5ff5ce691344c1e06fbe9ec1c997f9bf 2024-11-16T14:32:41,346 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 5ff5ce691344c1e06fbe9ec1c997f9bf; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60556181, jitterRate=-0.09764258563518524}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-16T14:32:41,346 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 5ff5ce691344c1e06fbe9ec1c997f9bf: Writing region info on filesystem at 1731767561339Initializing all the Stores at 1731767561340 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731767561340Cleaning up temporary data from old regions at 1731767561344 (+4 ms)Region opened successfully at 1731767561346 (+2 ms) 2024-11-16T14:32:41,355 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 5ff5ce691344c1e06fbe9ec1c997f9bf, disabling compactions & flushes 2024-11-16T14:32:41,355 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testDatalossWhenInputError,,1731767561265.5ff5ce691344c1e06fbe9ec1c997f9bf. 2024-11-16T14:32:41,355 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testDatalossWhenInputError,,1731767561265.5ff5ce691344c1e06fbe9ec1c997f9bf. 2024-11-16T14:32:41,355 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testDatalossWhenInputError,,1731767561265.5ff5ce691344c1e06fbe9ec1c997f9bf. after waiting 0 ms 2024-11-16T14:32:41,355 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testDatalossWhenInputError,,1731767561265.5ff5ce691344c1e06fbe9ec1c997f9bf. 2024-11-16T14:32:41,356 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 750 in region testDatalossWhenInputError,,1731767561265.5ff5ce691344c1e06fbe9ec1c997f9bf. 2024-11-16T14:32:41,356 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testDatalossWhenInputError,,1731767561265.5ff5ce691344c1e06fbe9ec1c997f9bf. 2024-11-16T14:32:41,356 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 5ff5ce691344c1e06fbe9ec1c997f9bf: Waiting for close lock at 1731767561355Disabling compacts and flushes for region at 1731767561355Disabling writes for close at 1731767561355Writing region close event to WAL at 1731767561356 (+1 ms)Closed at 1731767561356 2024-11-16T14:32:41,358 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33005 is added to blk_1073741911_1089 (size=1050) 2024-11-16T14:32:41,358 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36193 is added to blk_1073741911_1089 (size=1050) 2024-11-16T14:32:41,358 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43279 is added to blk_1073741911_1089 (size=1050) 2024-11-16T14:32:41,376 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:43457/hbase/WALs/testdatalosswheninputerror-manual,16010,1731767561264/wal.1731767561323, size=1.0 K (1050bytes) 2024-11-16T14:32:41,376 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:43457/hbase/WALs/testdatalosswheninputerror-manual,16010,1731767561264/wal.1731767561323 2024-11-16T14:32:41,376 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:43457/hbase/WALs/testdatalosswheninputerror-manual,16010,1731767561264/wal.1731767561323 after 0ms 2024-11-16T14:32:41,378 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:43457/hbase/WALs/testdatalosswheninputerror-manual,16010,1731767561264/wal.1731767561323: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-16T14:32:41,378 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:43457/hbase/WALs/testdatalosswheninputerror-manual,16010,1731767561264/wal.1731767561323 took 3ms 2024-11-16T14:32:41,380 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:43457/hbase/WALs/testdatalosswheninputerror-manual,16010,1731767561264/wal.1731767561323 so closing down 2024-11-16T14:32:41,380 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-16T14:32:41,381 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1731767561323.temp 2024-11-16T14:32:41,382 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:43457/hbase/data/default/testDatalossWhenInputError/5ff5ce691344c1e06fbe9ec1c997f9bf/recovered.edits/0000000000000000003-wal.1731767561323.temp 2024-11-16T14:32:41,383 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-16T14:32:41,389 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36193 is added to blk_1073741912_1090 (size=1050) 2024-11-16T14:32:41,390 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43279 is added to blk_1073741912_1090 (size=1050) 2024-11-16T14:32:41,390 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33005 is added to blk_1073741912_1090 (size=1050) 2024-11-16T14:32:41,390 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:43457/hbase/data/default/testDatalossWhenInputError/5ff5ce691344c1e06fbe9ec1c997f9bf/recovered.edits/0000000000000000003-wal.1731767561323.temp (wrote 10 edits, skipped 0 edits in 0 ms) 2024-11-16T14:32:41,392 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:43457/hbase/data/default/testDatalossWhenInputError/5ff5ce691344c1e06fbe9ec1c997f9bf/recovered.edits/0000000000000000003-wal.1731767561323.temp to hdfs://localhost:43457/hbase/data/default/testDatalossWhenInputError/5ff5ce691344c1e06fbe9ec1c997f9bf/recovered.edits/0000000000000000012 2024-11-16T14:32:41,392 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 10 edits across 1 Regions in 13 ms; skipped=0; WAL=hdfs://localhost:43457/hbase/WALs/testdatalosswheninputerror-manual,16010,1731767561264/wal.1731767561323, size=1.0 K, length=1050, corrupted=false, cancelled=false 2024-11-16T14:32:41,392 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:43457/hbase/WALs/testdatalosswheninputerror-manual,16010,1731767561264/wal.1731767561323, journal: Splitting hdfs://localhost:43457/hbase/WALs/testdatalosswheninputerror-manual,16010,1731767561264/wal.1731767561323, size=1.0 K (1050bytes) at 1731767561376Finishing writing output for hdfs://localhost:43457/hbase/WALs/testdatalosswheninputerror-manual,16010,1731767561264/wal.1731767561323 so closing down at 1731767561380 (+4 ms)Creating recovered edits writer path=hdfs://localhost:43457/hbase/data/default/testDatalossWhenInputError/5ff5ce691344c1e06fbe9ec1c997f9bf/recovered.edits/0000000000000000003-wal.1731767561323.temp at 1731767561382 (+2 ms)3 split writer threads finished at 1731767561383 (+1 ms)Closed recovered edits writer path=hdfs://localhost:43457/hbase/data/default/testDatalossWhenInputError/5ff5ce691344c1e06fbe9ec1c997f9bf/recovered.edits/0000000000000000003-wal.1731767561323.temp (wrote 10 edits, skipped 0 edits in 0 ms) at 1731767561390 (+7 ms)Rename recovered edits hdfs://localhost:43457/hbase/data/default/testDatalossWhenInputError/5ff5ce691344c1e06fbe9ec1c997f9bf/recovered.edits/0000000000000000003-wal.1731767561323.temp to hdfs://localhost:43457/hbase/data/default/testDatalossWhenInputError/5ff5ce691344c1e06fbe9ec1c997f9bf/recovered.edits/0000000000000000012 at 1731767561392 (+2 ms)Processed 10 edits across 1 Regions in 13 ms; skipped=0; WAL=hdfs://localhost:43457/hbase/WALs/testdatalosswheninputerror-manual,16010,1731767561264/wal.1731767561323, size=1.0 K, length=1050, corrupted=false, cancelled=false at 1731767561392 2024-11-16T14:32:41,394 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:43457/hbase/WALs/testdatalosswheninputerror-manual,16010,1731767561264/wal.1731767561323 to hdfs://localhost:43457/hbase/oldWALs/wal.1731767561323 2024-11-16T14:32:41,395 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:43457/hbase/data/default/testDatalossWhenInputError/5ff5ce691344c1e06fbe9ec1c997f9bf/recovered.edits/0000000000000000012 2024-11-16T14:32:41,397 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:43457/hbase/data/default/testDatalossWhenInputError/5ff5ce691344c1e06fbe9ec1c997f9bf/recovered.edits/0000000000000000012: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-16T14:32:41,736 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-16T14:32:41,738 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:43457/hbase/WALs/testdatalosswheninputerror-manual,16010,1731767561264, archiveDir=hdfs://localhost:43457/hbase/oldWALs, maxLogs=32 2024-11-16T14:32:41,750 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testdatalosswheninputerror-manual,16010,1731767561264/wal.1731767561739, exclude list is [], retry=0 2024-11-16T14:32:41,752 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36193,DS-a30e140c-8532-4096-8cae-c8d713a3d4e3,DISK] 2024-11-16T14:32:41,753 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43279,DS-17ab9554-4509-49b3-918b-178692b9a473,DISK] 2024-11-16T14:32:41,753 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33005,DS-6a1dd528-c0be-4580-a53e-7113879f7f97,DISK] 2024-11-16T14:32:41,754 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testdatalosswheninputerror-manual,16010,1731767561264/wal.1731767561739 2024-11-16T14:32:41,754 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35677:35677),(127.0.0.1/127.0.0.1:36015:36015),(127.0.0.1/127.0.0.1:41179:41179)] 2024-11-16T14:32:41,755 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 5ff5ce691344c1e06fbe9ec1c997f9bf, NAME => 'testDatalossWhenInputError,,1731767561265.5ff5ce691344c1e06fbe9ec1c997f9bf.', STARTKEY => '', ENDKEY => ''} 2024-11-16T14:32:41,755 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1731767561265.5ff5ce691344c1e06fbe9ec1c997f9bf.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T14:32:41,755 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 5ff5ce691344c1e06fbe9ec1c997f9bf 2024-11-16T14:32:41,755 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 5ff5ce691344c1e06fbe9ec1c997f9bf 2024-11-16T14:32:41,758 INFO [StoreOpener-5ff5ce691344c1e06fbe9ec1c997f9bf-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 5ff5ce691344c1e06fbe9ec1c997f9bf 2024-11-16T14:32:41,760 INFO [StoreOpener-5ff5ce691344c1e06fbe9ec1c997f9bf-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5ff5ce691344c1e06fbe9ec1c997f9bf columnFamilyName a 2024-11-16T14:32:41,760 DEBUG [StoreOpener-5ff5ce691344c1e06fbe9ec1c997f9bf-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:32:41,761 INFO [StoreOpener-5ff5ce691344c1e06fbe9ec1c997f9bf-1 {}] regionserver.HStore(327): Store=5ff5ce691344c1e06fbe9ec1c997f9bf/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T14:32:41,761 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 5ff5ce691344c1e06fbe9ec1c997f9bf 2024-11-16T14:32:41,761 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43457/hbase/data/default/testDatalossWhenInputError/5ff5ce691344c1e06fbe9ec1c997f9bf 2024-11-16T14:32:41,763 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:43457/hbase/data/default/testDatalossWhenInputError/5ff5ce691344c1e06fbe9ec1c997f9bf 2024-11-16T14:32:41,763 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:43457/hbase/data/default/testDatalossWhenInputError/5ff5ce691344c1e06fbe9ec1c997f9bf/recovered.edits/0000000000000000012 2024-11-16T14:32:41,765 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:43457/hbase/data/default/testDatalossWhenInputError/5ff5ce691344c1e06fbe9ec1c997f9bf/recovered.edits/0000000000000000012: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-16T14:32:41,766 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 10, skipped 0, firstSequenceIdInLog=3, maxSequenceIdInLog=12, path=hdfs://localhost:43457/hbase/data/default/testDatalossWhenInputError/5ff5ce691344c1e06fbe9ec1c997f9bf/recovered.edits/0000000000000000012 2024-11-16T14:32:41,766 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 5ff5ce691344c1e06fbe9ec1c997f9bf 1/1 column families, dataSize=750 B heapSize=1.73 KB 2024-11-16T14:32:41,784 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43457/hbase/data/default/testDatalossWhenInputError/5ff5ce691344c1e06fbe9ec1c997f9bf/.tmp/a/65a687d08411402e85370b2bf1f3e3b6 is 79, key is testDatalossWhenInputError/a:x0/1731767561346/Put/seqid=0 2024-11-16T14:32:41,791 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36193 is added to blk_1073741914_1092 (size=5808) 2024-11-16T14:32:41,791 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33005 is added to blk_1073741914_1092 (size=5808) 2024-11-16T14:32:41,792 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43279 is added to blk_1073741914_1092 (size=5808) 2024-11-16T14:32:41,792 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=750 B at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:43457/hbase/data/default/testDatalossWhenInputError/5ff5ce691344c1e06fbe9ec1c997f9bf/.tmp/a/65a687d08411402e85370b2bf1f3e3b6 2024-11-16T14:32:41,801 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43457/hbase/data/default/testDatalossWhenInputError/5ff5ce691344c1e06fbe9ec1c997f9bf/.tmp/a/65a687d08411402e85370b2bf1f3e3b6 as hdfs://localhost:43457/hbase/data/default/testDatalossWhenInputError/5ff5ce691344c1e06fbe9ec1c997f9bf/a/65a687d08411402e85370b2bf1f3e3b6 2024-11-16T14:32:41,809 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43457/hbase/data/default/testDatalossWhenInputError/5ff5ce691344c1e06fbe9ec1c997f9bf/a/65a687d08411402e85370b2bf1f3e3b6, entries=10, sequenceid=12, filesize=5.7 K 2024-11-16T14:32:41,809 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~750 B/750, heapSize ~1.72 KB/1760, currentSize=0 B/0 for 5ff5ce691344c1e06fbe9ec1c997f9bf in 43ms, sequenceid=12, compaction requested=false; wal=null 2024-11-16T14:32:41,810 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:43457/hbase/data/default/testDatalossWhenInputError/5ff5ce691344c1e06fbe9ec1c997f9bf/recovered.edits/0000000000000000012 2024-11-16T14:32:41,811 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 5ff5ce691344c1e06fbe9ec1c997f9bf 2024-11-16T14:32:41,811 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 5ff5ce691344c1e06fbe9ec1c997f9bf 2024-11-16T14:32:41,814 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 5ff5ce691344c1e06fbe9ec1c997f9bf 2024-11-16T14:32:41,817 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43457/hbase/data/default/testDatalossWhenInputError/5ff5ce691344c1e06fbe9ec1c997f9bf/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=1 2024-11-16T14:32:41,818 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 5ff5ce691344c1e06fbe9ec1c997f9bf; next sequenceid=13; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61149252, jitterRate=-0.08880513906478882}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-16T14:32:41,819 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 5ff5ce691344c1e06fbe9ec1c997f9bf: Writing region info on filesystem at 1731767561755Initializing all the Stores at 1731767561758 (+3 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731767561758Obtaining lock to block concurrent updates at 1731767561766 (+8 ms)Preparing flush snapshotting stores in 5ff5ce691344c1e06fbe9ec1c997f9bf at 1731767561766Finished memstore snapshotting testDatalossWhenInputError,,1731767561265.5ff5ce691344c1e06fbe9ec1c997f9bf., syncing WAL and waiting on mvcc, flushsize=dataSize=750, getHeapSize=1760, getOffHeapSize=0, getCellsCount=10 at 1731767561766Flushing stores of testDatalossWhenInputError,,1731767561265.5ff5ce691344c1e06fbe9ec1c997f9bf. at 1731767561766Flushing 5ff5ce691344c1e06fbe9ec1c997f9bf/a: creating writer at 1731767561766Flushing 5ff5ce691344c1e06fbe9ec1c997f9bf/a: appending metadata at 1731767561784 (+18 ms)Flushing 5ff5ce691344c1e06fbe9ec1c997f9bf/a: closing flushed file at 1731767561784Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@21ce98dd: reopening flushed file at 1731767561800 (+16 ms)Finished flush of dataSize ~750 B/750, heapSize ~1.72 KB/1760, currentSize=0 B/0 for 5ff5ce691344c1e06fbe9ec1c997f9bf in 43ms, sequenceid=12, compaction requested=false; wal=null at 1731767561809 (+9 ms)Cleaning up temporary data from old regions at 1731767561811 (+2 ms)Region opened successfully at 1731767561819 (+8 ms) 2024-11-16T14:32:41,823 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 5ff5ce691344c1e06fbe9ec1c997f9bf, NAME => 'testDatalossWhenInputError,,1731767561265.5ff5ce691344c1e06fbe9ec1c997f9bf.', STARTKEY => '', ENDKEY => ''} 2024-11-16T14:32:41,823 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1731767561265.5ff5ce691344c1e06fbe9ec1c997f9bf.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T14:32:41,824 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 5ff5ce691344c1e06fbe9ec1c997f9bf 2024-11-16T14:32:41,824 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 5ff5ce691344c1e06fbe9ec1c997f9bf 2024-11-16T14:32:41,825 INFO [StoreOpener-5ff5ce691344c1e06fbe9ec1c997f9bf-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 5ff5ce691344c1e06fbe9ec1c997f9bf 2024-11-16T14:32:41,826 INFO [StoreOpener-5ff5ce691344c1e06fbe9ec1c997f9bf-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5ff5ce691344c1e06fbe9ec1c997f9bf columnFamilyName a 2024-11-16T14:32:41,826 DEBUG [StoreOpener-5ff5ce691344c1e06fbe9ec1c997f9bf-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:32:41,833 DEBUG [StoreOpener-5ff5ce691344c1e06fbe9ec1c997f9bf-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:43457/hbase/data/default/testDatalossWhenInputError/5ff5ce691344c1e06fbe9ec1c997f9bf/a/65a687d08411402e85370b2bf1f3e3b6 2024-11-16T14:32:41,833 INFO [StoreOpener-5ff5ce691344c1e06fbe9ec1c997f9bf-1 {}] regionserver.HStore(327): Store=5ff5ce691344c1e06fbe9ec1c997f9bf/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T14:32:41,833 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 5ff5ce691344c1e06fbe9ec1c997f9bf 2024-11-16T14:32:41,834 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43457/hbase/data/default/testDatalossWhenInputError/5ff5ce691344c1e06fbe9ec1c997f9bf 2024-11-16T14:32:41,835 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43457/hbase/data/default/testDatalossWhenInputError/5ff5ce691344c1e06fbe9ec1c997f9bf 2024-11-16T14:32:41,836 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 5ff5ce691344c1e06fbe9ec1c997f9bf 2024-11-16T14:32:41,836 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 5ff5ce691344c1e06fbe9ec1c997f9bf 2024-11-16T14:32:41,838 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 5ff5ce691344c1e06fbe9ec1c997f9bf 2024-11-16T14:32:41,841 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43457/hbase/data/default/testDatalossWhenInputError/5ff5ce691344c1e06fbe9ec1c997f9bf/recovered.edits/13.seqid, newMaxSeqId=13, maxSeqId=12 2024-11-16T14:32:41,842 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 5ff5ce691344c1e06fbe9ec1c997f9bf; next sequenceid=14; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68470322, jitterRate=0.020287305116653442}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-16T14:32:41,842 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 5ff5ce691344c1e06fbe9ec1c997f9bf: Writing region info on filesystem at 1731767561824Initializing all the Stores at 1731767561825 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731767561825Cleaning up temporary data from old regions at 1731767561836 (+11 ms)Region opened successfully at 1731767561842 (+6 ms) 2024-11-16T14:32:41,860 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayValueCompression#testDatalossWhenInputError Thread=429 (was 419) Potentially hanging thread: AsyncFSWAL-24-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-24-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-865919460-172.17.0.2-1731767527819:blk_1073741913_1091, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1995475656_22 at /127.0.0.1:48742 [Receiving block BP-865919460-172.17.0.2-1731767527819:blk_1073741913_1091] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1995475656_22 at /127.0.0.1:39240 [Waiting for operation #16] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1995475656_22 at /127.0.0.1:54366 [Receiving block BP-865919460-172.17.0.2-1731767527819:blk_1073741913_1091] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1995475656_22 at /127.0.0.1:54314 [Waiting for operation #15] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-865919460-172.17.0.2-1731767527819:blk_1073741913_1091, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-865919460-172.17.0.2-1731767527819:blk_1073741913_1091, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1995475656_22 at /127.0.0.1:39312 [Receiving block BP-865919460-172.17.0.2-1731767527819:blk_1073741913_1091] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-24-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1264 (was 1183) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=142 (was 142), ProcessCount=11 (was 11), AvailableMemoryMB=4442 (was 4450) 2024-11-16T14:32:41,860 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1264 is superior to 1024 2024-11-16T14:32:41,871 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayValueCompression#testCompactedBulkLoadedFiles Thread=429, OpenFileDescriptor=1264, MaxFileDescriptor=1048576, SystemLoadAverage=142, ProcessCount=11, AvailableMemoryMB=4442 2024-11-16T14:32:41,871 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1264 is superior to 1024 2024-11-16T14:32:41,886 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-16T14:32:41,888 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-16T14:32:41,889 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-16T14:32:41,891 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-74092864, suffix=, logDir=hdfs://localhost:43457/hbase/WALs/hregion-74092864, archiveDir=hdfs://localhost:43457/hbase/oldWALs, maxLogs=32 2024-11-16T14:32:41,908 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-74092864/hregion-74092864.1731767561892, exclude list is [], retry=0 2024-11-16T14:32:41,911 DEBUG [AsyncFSWAL-26-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33005,DS-6a1dd528-c0be-4580-a53e-7113879f7f97,DISK] 2024-11-16T14:32:41,911 DEBUG [AsyncFSWAL-26-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43279,DS-17ab9554-4509-49b3-918b-178692b9a473,DISK] 2024-11-16T14:32:41,912 DEBUG [AsyncFSWAL-26-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36193,DS-a30e140c-8532-4096-8cae-c8d713a3d4e3,DISK] 2024-11-16T14:32:41,914 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-74092864/hregion-74092864.1731767561892 2024-11-16T14:32:41,914 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41179:41179),(127.0.0.1/127.0.0.1:36015:36015),(127.0.0.1/127.0.0.1:35677:35677)] 2024-11-16T14:32:41,914 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 8d5b640ddf71fb706e1914be3c6fed83, NAME => 'testCompactedBulkLoadedFiles,,1731767561887.8d5b640ddf71fb706e1914be3c6fed83.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testCompactedBulkLoadedFiles', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43457/hbase 2024-11-16T14:32:41,924 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43279 is added to blk_1073741916_1094 (size=63) 2024-11-16T14:32:41,924 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33005 is added to blk_1073741916_1094 (size=63) 2024-11-16T14:32:41,924 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36193 is added to blk_1073741916_1094 (size=63) 2024-11-16T14:32:41,925 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testCompactedBulkLoadedFiles,,1731767561887.8d5b640ddf71fb706e1914be3c6fed83.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T14:32:41,926 INFO [StoreOpener-8d5b640ddf71fb706e1914be3c6fed83-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 8d5b640ddf71fb706e1914be3c6fed83 2024-11-16T14:32:41,927 INFO [StoreOpener-8d5b640ddf71fb706e1914be3c6fed83-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8d5b640ddf71fb706e1914be3c6fed83 columnFamilyName a 2024-11-16T14:32:41,928 DEBUG [StoreOpener-8d5b640ddf71fb706e1914be3c6fed83-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:32:41,928 INFO [StoreOpener-8d5b640ddf71fb706e1914be3c6fed83-1 {}] regionserver.HStore(327): Store=8d5b640ddf71fb706e1914be3c6fed83/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T14:32:41,928 INFO [StoreOpener-8d5b640ddf71fb706e1914be3c6fed83-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 8d5b640ddf71fb706e1914be3c6fed83 2024-11-16T14:32:41,929 INFO [StoreOpener-8d5b640ddf71fb706e1914be3c6fed83-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8d5b640ddf71fb706e1914be3c6fed83 columnFamilyName b 2024-11-16T14:32:41,929 DEBUG [StoreOpener-8d5b640ddf71fb706e1914be3c6fed83-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:32:41,930 INFO [StoreOpener-8d5b640ddf71fb706e1914be3c6fed83-1 {}] regionserver.HStore(327): Store=8d5b640ddf71fb706e1914be3c6fed83/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T14:32:41,930 INFO [StoreOpener-8d5b640ddf71fb706e1914be3c6fed83-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 8d5b640ddf71fb706e1914be3c6fed83 2024-11-16T14:32:41,931 INFO [StoreOpener-8d5b640ddf71fb706e1914be3c6fed83-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8d5b640ddf71fb706e1914be3c6fed83 columnFamilyName c 2024-11-16T14:32:41,931 DEBUG [StoreOpener-8d5b640ddf71fb706e1914be3c6fed83-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:32:41,932 INFO [StoreOpener-8d5b640ddf71fb706e1914be3c6fed83-1 {}] regionserver.HStore(327): Store=8d5b640ddf71fb706e1914be3c6fed83/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T14:32:41,932 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 8d5b640ddf71fb706e1914be3c6fed83 2024-11-16T14:32:41,933 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43457/hbase/data/default/testCompactedBulkLoadedFiles/8d5b640ddf71fb706e1914be3c6fed83 2024-11-16T14:32:41,933 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43457/hbase/data/default/testCompactedBulkLoadedFiles/8d5b640ddf71fb706e1914be3c6fed83 2024-11-16T14:32:41,934 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 8d5b640ddf71fb706e1914be3c6fed83 2024-11-16T14:32:41,934 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 8d5b640ddf71fb706e1914be3c6fed83 2024-11-16T14:32:41,934 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testCompactedBulkLoadedFiles descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-16T14:32:41,936 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 8d5b640ddf71fb706e1914be3c6fed83 2024-11-16T14:32:41,938 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43457/hbase/data/default/testCompactedBulkLoadedFiles/8d5b640ddf71fb706e1914be3c6fed83/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-16T14:32:41,938 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 8d5b640ddf71fb706e1914be3c6fed83; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59435687, jitterRate=-0.11433924734592438}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-16T14:32:41,939 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 8d5b640ddf71fb706e1914be3c6fed83: Writing region info on filesystem at 1731767561925Initializing all the Stores at 1731767561926 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731767561926Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731767561926Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731767561926Cleaning up temporary data from old regions at 1731767561934 (+8 ms)Region opened successfully at 1731767561939 (+5 ms) 2024-11-16T14:32:41,939 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 8d5b640ddf71fb706e1914be3c6fed83, disabling compactions & flushes 2024-11-16T14:32:41,939 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testCompactedBulkLoadedFiles,,1731767561887.8d5b640ddf71fb706e1914be3c6fed83. 2024-11-16T14:32:41,939 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testCompactedBulkLoadedFiles,,1731767561887.8d5b640ddf71fb706e1914be3c6fed83. 2024-11-16T14:32:41,939 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testCompactedBulkLoadedFiles,,1731767561887.8d5b640ddf71fb706e1914be3c6fed83. after waiting 0 ms 2024-11-16T14:32:41,939 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testCompactedBulkLoadedFiles,,1731767561887.8d5b640ddf71fb706e1914be3c6fed83. 2024-11-16T14:32:41,940 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testCompactedBulkLoadedFiles,,1731767561887.8d5b640ddf71fb706e1914be3c6fed83. 2024-11-16T14:32:41,940 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 8d5b640ddf71fb706e1914be3c6fed83: Waiting for close lock at 1731767561939Disabling compacts and flushes for region at 1731767561939Disabling writes for close at 1731767561939Writing region close event to WAL at 1731767561940 (+1 ms)Closed at 1731767561940 2024-11-16T14:32:41,942 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36193 is added to blk_1073741915_1093 (size=95) 2024-11-16T14:32:41,942 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43279 is added to blk_1073741915_1093 (size=95) 2024-11-16T14:32:41,942 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33005 is added to blk_1073741915_1093 (size=95) 2024-11-16T14:32:41,945 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-16T14:32:41,945 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-74092864:(num 1731767561892) 2024-11-16T14:32:41,945 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-16T14:32:41,947 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:43457/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731767561886, archiveDir=hdfs://localhost:43457/hbase/oldWALs, maxLogs=32 2024-11-16T14:32:41,962 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731767561886/wal.1731767561947, exclude list is [], retry=0 2024-11-16T14:32:41,965 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43279,DS-17ab9554-4509-49b3-918b-178692b9a473,DISK] 2024-11-16T14:32:41,965 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33005,DS-6a1dd528-c0be-4580-a53e-7113879f7f97,DISK] 2024-11-16T14:32:41,965 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36193,DS-a30e140c-8532-4096-8cae-c8d713a3d4e3,DISK] 2024-11-16T14:32:41,967 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731767561886/wal.1731767561947 2024-11-16T14:32:41,967 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36015:36015),(127.0.0.1/127.0.0.1:41179:41179),(127.0.0.1/127.0.0.1:35677:35677)] 2024-11-16T14:32:41,967 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 8d5b640ddf71fb706e1914be3c6fed83, NAME => 'testCompactedBulkLoadedFiles,,1731767561887.8d5b640ddf71fb706e1914be3c6fed83.', STARTKEY => '', ENDKEY => ''} 2024-11-16T14:32:41,967 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testCompactedBulkLoadedFiles,,1731767561887.8d5b640ddf71fb706e1914be3c6fed83.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T14:32:41,968 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 8d5b640ddf71fb706e1914be3c6fed83 2024-11-16T14:32:41,968 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 8d5b640ddf71fb706e1914be3c6fed83 2024-11-16T14:32:41,969 INFO [StoreOpener-8d5b640ddf71fb706e1914be3c6fed83-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 8d5b640ddf71fb706e1914be3c6fed83 2024-11-16T14:32:41,970 INFO [StoreOpener-8d5b640ddf71fb706e1914be3c6fed83-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8d5b640ddf71fb706e1914be3c6fed83 columnFamilyName a 2024-11-16T14:32:41,970 DEBUG [StoreOpener-8d5b640ddf71fb706e1914be3c6fed83-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:32:41,971 INFO [StoreOpener-8d5b640ddf71fb706e1914be3c6fed83-1 {}] regionserver.HStore(327): Store=8d5b640ddf71fb706e1914be3c6fed83/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T14:32:41,971 INFO [StoreOpener-8d5b640ddf71fb706e1914be3c6fed83-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 8d5b640ddf71fb706e1914be3c6fed83 2024-11-16T14:32:41,972 INFO [StoreOpener-8d5b640ddf71fb706e1914be3c6fed83-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8d5b640ddf71fb706e1914be3c6fed83 columnFamilyName b 2024-11-16T14:32:41,972 DEBUG [StoreOpener-8d5b640ddf71fb706e1914be3c6fed83-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:32:41,973 INFO [StoreOpener-8d5b640ddf71fb706e1914be3c6fed83-1 {}] regionserver.HStore(327): Store=8d5b640ddf71fb706e1914be3c6fed83/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T14:32:41,973 INFO [StoreOpener-8d5b640ddf71fb706e1914be3c6fed83-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 8d5b640ddf71fb706e1914be3c6fed83 2024-11-16T14:32:41,973 INFO [StoreOpener-8d5b640ddf71fb706e1914be3c6fed83-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8d5b640ddf71fb706e1914be3c6fed83 columnFamilyName c 2024-11-16T14:32:41,974 DEBUG [StoreOpener-8d5b640ddf71fb706e1914be3c6fed83-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:32:41,974 INFO [StoreOpener-8d5b640ddf71fb706e1914be3c6fed83-1 {}] regionserver.HStore(327): Store=8d5b640ddf71fb706e1914be3c6fed83/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T14:32:41,974 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 8d5b640ddf71fb706e1914be3c6fed83 2024-11-16T14:32:41,975 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43457/hbase/data/default/testCompactedBulkLoadedFiles/8d5b640ddf71fb706e1914be3c6fed83 2024-11-16T14:32:41,976 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43457/hbase/data/default/testCompactedBulkLoadedFiles/8d5b640ddf71fb706e1914be3c6fed83 2024-11-16T14:32:41,977 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 8d5b640ddf71fb706e1914be3c6fed83 2024-11-16T14:32:41,977 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 8d5b640ddf71fb706e1914be3c6fed83 2024-11-16T14:32:41,977 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testCompactedBulkLoadedFiles descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-16T14:32:41,979 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 8d5b640ddf71fb706e1914be3c6fed83 2024-11-16T14:32:41,980 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 8d5b640ddf71fb706e1914be3c6fed83; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63817530, jitterRate=-0.04904469847679138}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-16T14:32:41,981 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 8d5b640ddf71fb706e1914be3c6fed83: Writing region info on filesystem at 1731767561968Initializing all the Stores at 1731767561969 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731767561969Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731767561969Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731767561969Cleaning up temporary data from old regions at 1731767561977 (+8 ms)Region opened successfully at 1731767561981 (+4 ms) 2024-11-16T14:32:41,985 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43457/hbase/testCompactedBulkLoadedFiles/hfile0 is 32, key is 000/a:a/1731767561985/Put/seqid=0 2024-11-16T14:32:41,994 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33005 is added to blk_1073741918_1096 (size=4875) 2024-11-16T14:32:41,994 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36193 is added to blk_1073741918_1096 (size=4875) 2024-11-16T14:32:41,994 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43279 is added to blk_1073741918_1096 (size=4875) 2024-11-16T14:32:41,996 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43457/hbase/testCompactedBulkLoadedFiles/hfile1 is 32, key is 100/a:a/1731767561996/Put/seqid=0 2024-11-16T14:32:42,005 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36193 is added to blk_1073741919_1097 (size=4875) 2024-11-16T14:32:42,005 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33005 is added to blk_1073741919_1097 (size=4875) 2024-11-16T14:32:42,005 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43279 is added to blk_1073741919_1097 (size=4875) 2024-11-16T14:32:42,007 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43457/hbase/testCompactedBulkLoadedFiles/hfile2 is 32, key is 200/a:a/1731767562007/Put/seqid=0 2024-11-16T14:32:42,012 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36193 is added to blk_1073741920_1098 (size=4875) 2024-11-16T14:32:42,013 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33005 is added to blk_1073741920_1098 (size=4875) 2024-11-16T14:32:42,013 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43279 is added to blk_1073741920_1098 (size=4875) 2024-11-16T14:32:42,016 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:43457/hbase/testCompactedBulkLoadedFiles/hfile0 for inclusion in 8d5b640ddf71fb706e1914be3c6fed83/a 2024-11-16T14:32:42,020 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first=000 last=050 2024-11-16T14:32:42,020 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-11-16T14:32:42,020 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:43457/hbase/testCompactedBulkLoadedFiles/hfile1 for inclusion in 8d5b640ddf71fb706e1914be3c6fed83/a 2024-11-16T14:32:42,023 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first=100 last=150 2024-11-16T14:32:42,024 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-11-16T14:32:42,024 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:43457/hbase/testCompactedBulkLoadedFiles/hfile2 for inclusion in 8d5b640ddf71fb706e1914be3c6fed83/a 2024-11-16T14:32:42,027 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first=200 last=250 2024-11-16T14:32:42,027 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-11-16T14:32:42,027 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 8d5b640ddf71fb706e1914be3c6fed83 3/3 column families, dataSize=51 B heapSize=896 B 2024-11-16T14:32:42,042 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43457/hbase/data/default/testCompactedBulkLoadedFiles/8d5b640ddf71fb706e1914be3c6fed83/.tmp/a/07a94e4c2075448c95ef9f6419778973 is 55, key is testCompactedBulkLoadedFiles/a:a/1731767561981/Put/seqid=0 2024-11-16T14:32:42,049 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43279 is added to blk_1073741921_1099 (size=5107) 2024-11-16T14:32:42,049 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33005 is added to blk_1073741921_1099 (size=5107) 2024-11-16T14:32:42,049 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36193 is added to blk_1073741921_1099 (size=5107) 2024-11-16T14:32:42,049 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51 B at sequenceid=4 (bloomFilter=true), to=hdfs://localhost:43457/hbase/data/default/testCompactedBulkLoadedFiles/8d5b640ddf71fb706e1914be3c6fed83/.tmp/a/07a94e4c2075448c95ef9f6419778973 2024-11-16T14:32:42,055 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43457/hbase/data/default/testCompactedBulkLoadedFiles/8d5b640ddf71fb706e1914be3c6fed83/.tmp/a/07a94e4c2075448c95ef9f6419778973 as hdfs://localhost:43457/hbase/data/default/testCompactedBulkLoadedFiles/8d5b640ddf71fb706e1914be3c6fed83/a/07a94e4c2075448c95ef9f6419778973 2024-11-16T14:32:42,060 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43457/hbase/data/default/testCompactedBulkLoadedFiles/8d5b640ddf71fb706e1914be3c6fed83/a/07a94e4c2075448c95ef9f6419778973, entries=1, sequenceid=4, filesize=5.0 K 2024-11-16T14:32:42,061 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~51 B/51, heapSize ~368 B/368, currentSize=0 B/0 for 8d5b640ddf71fb706e1914be3c6fed83 in 34ms, sequenceid=4, compaction requested=false 2024-11-16T14:32:42,061 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 8d5b640ddf71fb706e1914be3c6fed83: 2024-11-16T14:32:42,062 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43457/hbase/testCompactedBulkLoadedFiles/hfile0 as hdfs://localhost:43457/hbase/data/default/testCompactedBulkLoadedFiles/8d5b640ddf71fb706e1914be3c6fed83/a/d73e5af2830f4709bb6ddd023541dbf2_SeqId_4_ 2024-11-16T14:32:42,063 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43457/hbase/testCompactedBulkLoadedFiles/hfile1 as hdfs://localhost:43457/hbase/data/default/testCompactedBulkLoadedFiles/8d5b640ddf71fb706e1914be3c6fed83/a/cdfc007884514ee09675db693c0fd48f_SeqId_4_ 2024-11-16T14:32:42,064 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43457/hbase/testCompactedBulkLoadedFiles/hfile2 as hdfs://localhost:43457/hbase/data/default/testCompactedBulkLoadedFiles/8d5b640ddf71fb706e1914be3c6fed83/a/bb488e5c843d401589ce8e3789cdbb5a_SeqId_4_ 2024-11-16T14:32:42,065 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:43457/hbase/testCompactedBulkLoadedFiles/hfile0 into 8d5b640ddf71fb706e1914be3c6fed83/a as hdfs://localhost:43457/hbase/data/default/testCompactedBulkLoadedFiles/8d5b640ddf71fb706e1914be3c6fed83/a/d73e5af2830f4709bb6ddd023541dbf2_SeqId_4_ - updating store file list. 2024-11-16T14:32:42,068 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for d73e5af2830f4709bb6ddd023541dbf2_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-16T14:32:42,068 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:43457/hbase/data/default/testCompactedBulkLoadedFiles/8d5b640ddf71fb706e1914be3c6fed83/a/d73e5af2830f4709bb6ddd023541dbf2_SeqId_4_ into 8d5b640ddf71fb706e1914be3c6fed83/a 2024-11-16T14:32:42,068 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:43457/hbase/testCompactedBulkLoadedFiles/hfile0 into 8d5b640ddf71fb706e1914be3c6fed83/a (new location: hdfs://localhost:43457/hbase/data/default/testCompactedBulkLoadedFiles/8d5b640ddf71fb706e1914be3c6fed83/a/d73e5af2830f4709bb6ddd023541dbf2_SeqId_4_) 2024-11-16T14:32:42,069 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:43457/hbase/testCompactedBulkLoadedFiles/hfile1 into 8d5b640ddf71fb706e1914be3c6fed83/a as hdfs://localhost:43457/hbase/data/default/testCompactedBulkLoadedFiles/8d5b640ddf71fb706e1914be3c6fed83/a/cdfc007884514ee09675db693c0fd48f_SeqId_4_ - updating store file list. 2024-11-16T14:32:42,074 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for cdfc007884514ee09675db693c0fd48f_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-16T14:32:42,074 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:43457/hbase/data/default/testCompactedBulkLoadedFiles/8d5b640ddf71fb706e1914be3c6fed83/a/cdfc007884514ee09675db693c0fd48f_SeqId_4_ into 8d5b640ddf71fb706e1914be3c6fed83/a 2024-11-16T14:32:42,074 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:43457/hbase/testCompactedBulkLoadedFiles/hfile1 into 8d5b640ddf71fb706e1914be3c6fed83/a (new location: hdfs://localhost:43457/hbase/data/default/testCompactedBulkLoadedFiles/8d5b640ddf71fb706e1914be3c6fed83/a/cdfc007884514ee09675db693c0fd48f_SeqId_4_) 2024-11-16T14:32:42,075 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:43457/hbase/testCompactedBulkLoadedFiles/hfile2 into 8d5b640ddf71fb706e1914be3c6fed83/a as hdfs://localhost:43457/hbase/data/default/testCompactedBulkLoadedFiles/8d5b640ddf71fb706e1914be3c6fed83/a/bb488e5c843d401589ce8e3789cdbb5a_SeqId_4_ - updating store file list. 2024-11-16T14:32:42,079 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for bb488e5c843d401589ce8e3789cdbb5a_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-16T14:32:42,080 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:43457/hbase/data/default/testCompactedBulkLoadedFiles/8d5b640ddf71fb706e1914be3c6fed83/a/bb488e5c843d401589ce8e3789cdbb5a_SeqId_4_ into 8d5b640ddf71fb706e1914be3c6fed83/a 2024-11-16T14:32:42,080 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:43457/hbase/testCompactedBulkLoadedFiles/hfile2 into 8d5b640ddf71fb706e1914be3c6fed83/a (new location: hdfs://localhost:43457/hbase/data/default/testCompactedBulkLoadedFiles/8d5b640ddf71fb706e1914be3c6fed83/a/bb488e5c843d401589ce8e3789cdbb5a_SeqId_4_) 2024-11-16T14:32:42,086 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T14:32:42,086 DEBUG [Time-limited test {}] regionserver.HStore(1541): 8d5b640ddf71fb706e1914be3c6fed83/a is initiating major compaction (all files) 2024-11-16T14:32:42,087 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of 8d5b640ddf71fb706e1914be3c6fed83/a in testCompactedBulkLoadedFiles,,1731767561887.8d5b640ddf71fb706e1914be3c6fed83. 2024-11-16T14:32:42,087 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43457/hbase/data/default/testCompactedBulkLoadedFiles/8d5b640ddf71fb706e1914be3c6fed83/a/07a94e4c2075448c95ef9f6419778973, hdfs://localhost:43457/hbase/data/default/testCompactedBulkLoadedFiles/8d5b640ddf71fb706e1914be3c6fed83/a/d73e5af2830f4709bb6ddd023541dbf2_SeqId_4_, hdfs://localhost:43457/hbase/data/default/testCompactedBulkLoadedFiles/8d5b640ddf71fb706e1914be3c6fed83/a/cdfc007884514ee09675db693c0fd48f_SeqId_4_, hdfs://localhost:43457/hbase/data/default/testCompactedBulkLoadedFiles/8d5b640ddf71fb706e1914be3c6fed83/a/bb488e5c843d401589ce8e3789cdbb5a_SeqId_4_] into tmpdir=hdfs://localhost:43457/hbase/data/default/testCompactedBulkLoadedFiles/8d5b640ddf71fb706e1914be3c6fed83/.tmp, totalSize=19.3 K 2024-11-16T14:32:42,087 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 07a94e4c2075448c95ef9f6419778973, keycount=1, bloomtype=ROW, size=5.0 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=1731767561981 2024-11-16T14:32:42,088 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting d73e5af2830f4709bb6ddd023541dbf2_SeqId_4_, keycount=10, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=-9223372036854775808 2024-11-16T14:32:42,088 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting cdfc007884514ee09675db693c0fd48f_SeqId_4_, keycount=10, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=-9223372036854775808 2024-11-16T14:32:42,088 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting bb488e5c843d401589ce8e3789cdbb5a_SeqId_4_, keycount=10, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=-9223372036854775808 2024-11-16T14:32:42,100 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43457/hbase/data/default/testCompactedBulkLoadedFiles/8d5b640ddf71fb706e1914be3c6fed83/.tmp/a/e6044208413e45cfa9c058918bd53df3 is 55, key is testCompactedBulkLoadedFiles/a:a/1731767561981/Put/seqid=0 2024-11-16T14:32:42,105 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33005 is added to blk_1073741922_1100 (size=6154) 2024-11-16T14:32:42,105 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36193 is added to blk_1073741922_1100 (size=6154) 2024-11-16T14:32:42,106 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43279 is added to blk_1073741922_1100 (size=6154) 2024-11-16T14:32:42,111 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43457/hbase/data/default/testCompactedBulkLoadedFiles/8d5b640ddf71fb706e1914be3c6fed83/.tmp/a/e6044208413e45cfa9c058918bd53df3 as hdfs://localhost:43457/hbase/data/default/testCompactedBulkLoadedFiles/8d5b640ddf71fb706e1914be3c6fed83/a/e6044208413e45cfa9c058918bd53df3 2024-11-16T14:32:42,117 INFO [Time-limited test {}] regionserver.HStore(1337): Completed major compaction of 4 (all) file(s) in 8d5b640ddf71fb706e1914be3c6fed83/a of 8d5b640ddf71fb706e1914be3c6fed83 into e6044208413e45cfa9c058918bd53df3(size=6.0 K), total size for store is 6.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:32:42,117 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for 8d5b640ddf71fb706e1914be3c6fed83: 2024-11-16T14:32:42,117 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 0 store files, 0 compacting, 0 eligible, 16 blocking 2024-11-16T14:32:42,117 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 0 store files, 0 compacting, 0 eligible, 16 blocking 2024-11-16T14:32:42,147 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:43457/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731767561886/wal.1731767561947, size=0 (0bytes) 2024-11-16T14:32:42,147 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:43457/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731767561886/wal.1731767561947 might be still open, length is 0 2024-11-16T14:32:42,147 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:43457/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731767561886/wal.1731767561947 2024-11-16T14:32:42,147 WARN [IPC Server handler 3 on default port 43457 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731767561886/wal.1731767561947 has not been closed. Lease recovery is in progress. RecoveryId = 1101 for block blk_1073741917_1095 2024-11-16T14:32:42,148 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:43457/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731767561886/wal.1731767561947 after 1ms 2024-11-16T14:32:42,430 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testDatalossWhenInputError 2024-11-16T14:32:42,430 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testDatalossWhenInputError Metrics about Tables on a single HBase RegionServer 2024-11-16T14:32:42,431 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testCompactedBulkLoadedFiles 2024-11-16T14:32:42,431 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testCompactedBulkLoadedFiles Metrics about Tables on a single HBase RegionServer 2024-11-16T14:32:44,264 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1995475656_22 at /127.0.0.1:48300 [Receiving block BP-865919460-172.17.0.2-1731767527819:blk_1073741917_1095] {}] datanode.DataXceiver(331): 127.0.0.1:43279:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:48300 dst: /127.0.0.1:43279 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:43279 remote=/127.0.0.1:48300]. Total timeout mills is 60000, 57853 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:32:44,266 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1995475656_22 at /127.0.0.1:40646 [Receiving block BP-865919460-172.17.0.2-1731767527819:blk_1073741917_1095] {}] datanode.DataXceiver(331): 127.0.0.1:36193:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:40646 dst: /127.0.0.1:36193 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:32:44,266 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1995475656_22 at /127.0.0.1:54326 [Receiving block BP-865919460-172.17.0.2-1731767527819:blk_1073741917_1095] {}] datanode.DataXceiver(331): 127.0.0.1:33005:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:54326 dst: /127.0.0.1:33005 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:32:44,270 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43279 is added to blk_1073741917_1101 (size=1173) 2024-11-16T14:32:44,271 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33005 is added to blk_1073741917_1101 (size=1173) 2024-11-16T14:32:46,149 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:43457/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731767561886/wal.1731767561947 after 4002ms 2024-11-16T14:32:46,153 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:43457/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731767561886/wal.1731767561947: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-16T14:32:46,153 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:43457/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731767561886/wal.1731767561947 took 4007ms 2024-11-16T14:32:46,155 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:43457/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731767561886/wal.1731767561947; continuing. 2024-11-16T14:32:46,155 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:43457/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731767561886/wal.1731767561947 so closing down 2024-11-16T14:32:46,156 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-16T14:32:46,157 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1731767561947.temp 2024-11-16T14:32:46,159 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:43457/hbase/data/default/testCompactedBulkLoadedFiles/8d5b640ddf71fb706e1914be3c6fed83/recovered.edits/0000000000000000003-wal.1731767561947.temp 2024-11-16T14:32:46,160 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-16T14:32:46,168 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33005 is added to blk_1073741923_1102 (size=549) 2024-11-16T14:32:46,169 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36193 is added to blk_1073741923_1102 (size=549) 2024-11-16T14:32:46,169 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43279 is added to blk_1073741923_1102 (size=549) 2024-11-16T14:32:46,170 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:43457/hbase/data/default/testCompactedBulkLoadedFiles/8d5b640ddf71fb706e1914be3c6fed83/recovered.edits/0000000000000000003-wal.1731767561947.temp (wrote 2 edits, skipped 0 edits in 0 ms) 2024-11-16T14:32:46,172 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:43457/hbase/data/default/testCompactedBulkLoadedFiles/8d5b640ddf71fb706e1914be3c6fed83/recovered.edits/0000000000000000003-wal.1731767561947.temp to hdfs://localhost:43457/hbase/data/default/testCompactedBulkLoadedFiles/8d5b640ddf71fb706e1914be3c6fed83/recovered.edits/0000000000000000008 2024-11-16T14:32:46,172 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 5 edits across 1 Regions in 18 ms; skipped=3; WAL=hdfs://localhost:43457/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731767561886/wal.1731767561947, size=0, length=0, corrupted=false, cancelled=false 2024-11-16T14:32:46,172 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:43457/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731767561886/wal.1731767561947, journal: Splitting hdfs://localhost:43457/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731767561886/wal.1731767561947, size=0 (0bytes) at 1731767562147Finishing writing output for hdfs://localhost:43457/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731767561886/wal.1731767561947 so closing down at 1731767566155 (+4008 ms)Creating recovered edits writer path=hdfs://localhost:43457/hbase/data/default/testCompactedBulkLoadedFiles/8d5b640ddf71fb706e1914be3c6fed83/recovered.edits/0000000000000000003-wal.1731767561947.temp at 1731767566159 (+4 ms)3 split writer threads finished at 1731767566160 (+1 ms)Closed recovered edits writer path=hdfs://localhost:43457/hbase/data/default/testCompactedBulkLoadedFiles/8d5b640ddf71fb706e1914be3c6fed83/recovered.edits/0000000000000000003-wal.1731767561947.temp (wrote 2 edits, skipped 0 edits in 0 ms) at 1731767566170 (+10 ms)Rename recovered edits hdfs://localhost:43457/hbase/data/default/testCompactedBulkLoadedFiles/8d5b640ddf71fb706e1914be3c6fed83/recovered.edits/0000000000000000003-wal.1731767561947.temp to hdfs://localhost:43457/hbase/data/default/testCompactedBulkLoadedFiles/8d5b640ddf71fb706e1914be3c6fed83/recovered.edits/0000000000000000008 at 1731767566172 (+2 ms)Processed 5 edits across 1 Regions in 18 ms; skipped=3; WAL=hdfs://localhost:43457/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731767561886/wal.1731767561947, size=0, length=0, corrupted=false, cancelled=false at 1731767566172 2024-11-16T14:32:46,174 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:43457/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731767561886/wal.1731767561947 to hdfs://localhost:43457/hbase/oldWALs/wal.1731767561947 2024-11-16T14:32:46,175 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:43457/hbase/data/default/testCompactedBulkLoadedFiles/8d5b640ddf71fb706e1914be3c6fed83/recovered.edits/0000000000000000008 2024-11-16T14:32:46,175 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-16T14:32:46,177 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:43457/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731767561886, archiveDir=hdfs://localhost:43457/hbase/oldWALs, maxLogs=32 2024-11-16T14:32:46,189 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731767561886/wal.1731767566177, exclude list is [], retry=0 2024-11-16T14:32:46,191 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33005,DS-6a1dd528-c0be-4580-a53e-7113879f7f97,DISK] 2024-11-16T14:32:46,192 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36193,DS-a30e140c-8532-4096-8cae-c8d713a3d4e3,DISK] 2024-11-16T14:32:46,192 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43279,DS-17ab9554-4509-49b3-918b-178692b9a473,DISK] 2024-11-16T14:32:46,194 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731767561886/wal.1731767566177 2024-11-16T14:32:46,194 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41179:41179),(127.0.0.1/127.0.0.1:35677:35677),(127.0.0.1/127.0.0.1:36015:36015)] 2024-11-16T14:32:46,194 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 8d5b640ddf71fb706e1914be3c6fed83, NAME => 'testCompactedBulkLoadedFiles,,1731767561887.8d5b640ddf71fb706e1914be3c6fed83.', STARTKEY => '', ENDKEY => ''} 2024-11-16T14:32:46,194 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testCompactedBulkLoadedFiles,,1731767561887.8d5b640ddf71fb706e1914be3c6fed83.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T14:32:46,194 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 8d5b640ddf71fb706e1914be3c6fed83 2024-11-16T14:32:46,194 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 8d5b640ddf71fb706e1914be3c6fed83 2024-11-16T14:32:46,196 INFO [StoreOpener-8d5b640ddf71fb706e1914be3c6fed83-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 8d5b640ddf71fb706e1914be3c6fed83 2024-11-16T14:32:46,196 INFO [StoreOpener-8d5b640ddf71fb706e1914be3c6fed83-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8d5b640ddf71fb706e1914be3c6fed83 columnFamilyName a 2024-11-16T14:32:46,196 DEBUG [StoreOpener-8d5b640ddf71fb706e1914be3c6fed83-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:32:46,206 DEBUG [StoreOpener-8d5b640ddf71fb706e1914be3c6fed83-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:43457/hbase/data/default/testCompactedBulkLoadedFiles/8d5b640ddf71fb706e1914be3c6fed83/a/07a94e4c2075448c95ef9f6419778973 2024-11-16T14:32:46,209 DEBUG [StoreFileOpener-8d5b640ddf71fb706e1914be3c6fed83-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for bb488e5c843d401589ce8e3789cdbb5a_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-16T14:32:46,210 DEBUG [StoreOpener-8d5b640ddf71fb706e1914be3c6fed83-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:43457/hbase/data/default/testCompactedBulkLoadedFiles/8d5b640ddf71fb706e1914be3c6fed83/a/bb488e5c843d401589ce8e3789cdbb5a_SeqId_4_ 2024-11-16T14:32:46,213 DEBUG [StoreFileOpener-8d5b640ddf71fb706e1914be3c6fed83-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for cdfc007884514ee09675db693c0fd48f_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-16T14:32:46,213 DEBUG [StoreOpener-8d5b640ddf71fb706e1914be3c6fed83-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:43457/hbase/data/default/testCompactedBulkLoadedFiles/8d5b640ddf71fb706e1914be3c6fed83/a/cdfc007884514ee09675db693c0fd48f_SeqId_4_ 2024-11-16T14:32:46,217 DEBUG [StoreFileOpener-8d5b640ddf71fb706e1914be3c6fed83-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for d73e5af2830f4709bb6ddd023541dbf2_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-16T14:32:46,217 DEBUG [StoreOpener-8d5b640ddf71fb706e1914be3c6fed83-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:43457/hbase/data/default/testCompactedBulkLoadedFiles/8d5b640ddf71fb706e1914be3c6fed83/a/d73e5af2830f4709bb6ddd023541dbf2_SeqId_4_ 2024-11-16T14:32:46,220 DEBUG [StoreOpener-8d5b640ddf71fb706e1914be3c6fed83-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:43457/hbase/data/default/testCompactedBulkLoadedFiles/8d5b640ddf71fb706e1914be3c6fed83/a/e6044208413e45cfa9c058918bd53df3 2024-11-16T14:32:46,220 WARN [StoreOpener-8d5b640ddf71fb706e1914be3c6fed83-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:43457/hbase/data/default/testCompactedBulkLoadedFiles/8d5b640ddf71fb706e1914be3c6fed83/a/07a94e4c2075448c95ef9f6419778973 from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@7dd5dce9 2024-11-16T14:32:46,220 WARN [StoreOpener-8d5b640ddf71fb706e1914be3c6fed83-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:43457/hbase/data/default/testCompactedBulkLoadedFiles/8d5b640ddf71fb706e1914be3c6fed83/a/bb488e5c843d401589ce8e3789cdbb5a_SeqId_4_ from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@7dd5dce9 2024-11-16T14:32:46,220 WARN [StoreOpener-8d5b640ddf71fb706e1914be3c6fed83-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:43457/hbase/data/default/testCompactedBulkLoadedFiles/8d5b640ddf71fb706e1914be3c6fed83/a/cdfc007884514ee09675db693c0fd48f_SeqId_4_ from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@7dd5dce9 2024-11-16T14:32:46,221 WARN [StoreOpener-8d5b640ddf71fb706e1914be3c6fed83-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:43457/hbase/data/default/testCompactedBulkLoadedFiles/8d5b640ddf71fb706e1914be3c6fed83/a/d73e5af2830f4709bb6ddd023541dbf2_SeqId_4_ from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@7dd5dce9 2024-11-16T14:32:46,221 DEBUG [StoreOpener-8d5b640ddf71fb706e1914be3c6fed83-1 {}] regionserver.StoreEngine(327): Moving the files [hdfs://localhost:43457/hbase/data/default/testCompactedBulkLoadedFiles/8d5b640ddf71fb706e1914be3c6fed83/a/07a94e4c2075448c95ef9f6419778973, hdfs://localhost:43457/hbase/data/default/testCompactedBulkLoadedFiles/8d5b640ddf71fb706e1914be3c6fed83/a/bb488e5c843d401589ce8e3789cdbb5a_SeqId_4_, hdfs://localhost:43457/hbase/data/default/testCompactedBulkLoadedFiles/8d5b640ddf71fb706e1914be3c6fed83/a/cdfc007884514ee09675db693c0fd48f_SeqId_4_, hdfs://localhost:43457/hbase/data/default/testCompactedBulkLoadedFiles/8d5b640ddf71fb706e1914be3c6fed83/a/d73e5af2830f4709bb6ddd023541dbf2_SeqId_4_] to archive 2024-11-16T14:32:46,221 DEBUG [StoreOpener-8d5b640ddf71fb706e1914be3c6fed83-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-16T14:32:46,223 DEBUG [StoreOpener-8d5b640ddf71fb706e1914be3c6fed83-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43457/hbase/data/default/testCompactedBulkLoadedFiles/8d5b640ddf71fb706e1914be3c6fed83/a/07a94e4c2075448c95ef9f6419778973 to hdfs://localhost:43457/hbase/archive/data/default/testCompactedBulkLoadedFiles/8d5b640ddf71fb706e1914be3c6fed83/a/07a94e4c2075448c95ef9f6419778973 2024-11-16T14:32:46,224 DEBUG [StoreOpener-8d5b640ddf71fb706e1914be3c6fed83-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43457/hbase/data/default/testCompactedBulkLoadedFiles/8d5b640ddf71fb706e1914be3c6fed83/a/bb488e5c843d401589ce8e3789cdbb5a_SeqId_4_ to hdfs://localhost:43457/hbase/archive/data/default/testCompactedBulkLoadedFiles/8d5b640ddf71fb706e1914be3c6fed83/a/bb488e5c843d401589ce8e3789cdbb5a_SeqId_4_ 2024-11-16T14:32:46,225 DEBUG [StoreOpener-8d5b640ddf71fb706e1914be3c6fed83-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43457/hbase/data/default/testCompactedBulkLoadedFiles/8d5b640ddf71fb706e1914be3c6fed83/a/cdfc007884514ee09675db693c0fd48f_SeqId_4_ to hdfs://localhost:43457/hbase/archive/data/default/testCompactedBulkLoadedFiles/8d5b640ddf71fb706e1914be3c6fed83/a/cdfc007884514ee09675db693c0fd48f_SeqId_4_ 2024-11-16T14:32:46,226 DEBUG [StoreOpener-8d5b640ddf71fb706e1914be3c6fed83-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43457/hbase/data/default/testCompactedBulkLoadedFiles/8d5b640ddf71fb706e1914be3c6fed83/a/d73e5af2830f4709bb6ddd023541dbf2_SeqId_4_ to hdfs://localhost:43457/hbase/archive/data/default/testCompactedBulkLoadedFiles/8d5b640ddf71fb706e1914be3c6fed83/a/d73e5af2830f4709bb6ddd023541dbf2_SeqId_4_ 2024-11-16T14:32:46,226 INFO [StoreOpener-8d5b640ddf71fb706e1914be3c6fed83-1 {}] regionserver.HStore(327): Store=8d5b640ddf71fb706e1914be3c6fed83/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T14:32:46,226 INFO [StoreOpener-8d5b640ddf71fb706e1914be3c6fed83-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 8d5b640ddf71fb706e1914be3c6fed83 2024-11-16T14:32:46,227 INFO [StoreOpener-8d5b640ddf71fb706e1914be3c6fed83-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8d5b640ddf71fb706e1914be3c6fed83 columnFamilyName b 2024-11-16T14:32:46,227 DEBUG [StoreOpener-8d5b640ddf71fb706e1914be3c6fed83-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:32:46,227 INFO [StoreOpener-8d5b640ddf71fb706e1914be3c6fed83-1 {}] regionserver.HStore(327): Store=8d5b640ddf71fb706e1914be3c6fed83/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T14:32:46,227 INFO [StoreOpener-8d5b640ddf71fb706e1914be3c6fed83-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 8d5b640ddf71fb706e1914be3c6fed83 2024-11-16T14:32:46,228 INFO [StoreOpener-8d5b640ddf71fb706e1914be3c6fed83-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8d5b640ddf71fb706e1914be3c6fed83 columnFamilyName c 2024-11-16T14:32:46,228 DEBUG [StoreOpener-8d5b640ddf71fb706e1914be3c6fed83-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:32:46,228 INFO [StoreOpener-8d5b640ddf71fb706e1914be3c6fed83-1 {}] regionserver.HStore(327): Store=8d5b640ddf71fb706e1914be3c6fed83/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T14:32:46,228 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 8d5b640ddf71fb706e1914be3c6fed83 2024-11-16T14:32:46,229 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43457/hbase/data/default/testCompactedBulkLoadedFiles/8d5b640ddf71fb706e1914be3c6fed83 2024-11-16T14:32:46,230 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:43457/hbase/data/default/testCompactedBulkLoadedFiles/8d5b640ddf71fb706e1914be3c6fed83 2024-11-16T14:32:46,230 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:43457/hbase/data/default/testCompactedBulkLoadedFiles/8d5b640ddf71fb706e1914be3c6fed83/recovered.edits/0000000000000000008 2024-11-16T14:32:46,232 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:43457/hbase/data/default/testCompactedBulkLoadedFiles/8d5b640ddf71fb706e1914be3c6fed83/recovered.edits/0000000000000000008: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-16T14:32:46,233 DEBUG [Time-limited test {}] regionserver.HRegion(5836): 8d5b640ddf71fb706e1914be3c6fed83 : Replaying compaction marker table_name: "testCompactedBulkLoadedFiles" encoded_region_name: "8d5b640ddf71fb706e1914be3c6fed83" family_name: "a" compaction_input: "07a94e4c2075448c95ef9f6419778973" compaction_input: "d73e5af2830f4709bb6ddd023541dbf2_SeqId_4_" compaction_input: "cdfc007884514ee09675db693c0fd48f_SeqId_4_" compaction_input: "bb488e5c843d401589ce8e3789cdbb5a_SeqId_4_" compaction_output: "e6044208413e45cfa9c058918bd53df3" store_home_dir: "a" region_name: "testCompactedBulkLoadedFiles,,1731767561887.8d5b640ddf71fb706e1914be3c6fed83." with seqId=9223372036854775807 and lastReplayedOpenRegionSeqId=-1 2024-11-16T14:32:46,234 DEBUG [Time-limited test {}] regionserver.HStore(1354): Completing compaction from the WAL marker 2024-11-16T14:32:46,234 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 0, skipped 2, firstSequenceIdInLog=3, maxSequenceIdInLog=8, path=hdfs://localhost:43457/hbase/data/default/testCompactedBulkLoadedFiles/8d5b640ddf71fb706e1914be3c6fed83/recovered.edits/0000000000000000008 2024-11-16T14:32:46,234 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:43457/hbase/data/default/testCompactedBulkLoadedFiles/8d5b640ddf71fb706e1914be3c6fed83/recovered.edits/0000000000000000008 2024-11-16T14:32:46,235 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 8d5b640ddf71fb706e1914be3c6fed83 2024-11-16T14:32:46,235 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 8d5b640ddf71fb706e1914be3c6fed83 2024-11-16T14:32:46,236 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testCompactedBulkLoadedFiles descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-16T14:32:46,237 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 8d5b640ddf71fb706e1914be3c6fed83 2024-11-16T14:32:46,239 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43457/hbase/data/default/testCompactedBulkLoadedFiles/8d5b640ddf71fb706e1914be3c6fed83/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-16T14:32:46,240 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 8d5b640ddf71fb706e1914be3c6fed83; next sequenceid=9; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60660390, jitterRate=-0.09608975052833557}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-16T14:32:46,240 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 8d5b640ddf71fb706e1914be3c6fed83: Writing region info on filesystem at 1731767566194Initializing all the Stores at 1731767566195 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731767566195Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731767566195Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731767566196 (+1 ms)Cleaning up temporary data from old regions at 1731767566235 (+39 ms)Region opened successfully at 1731767566240 (+5 ms) 2024-11-16T14:32:46,243 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 8d5b640ddf71fb706e1914be3c6fed83, disabling compactions & flushes 2024-11-16T14:32:46,243 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testCompactedBulkLoadedFiles,,1731767561887.8d5b640ddf71fb706e1914be3c6fed83. 2024-11-16T14:32:46,243 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testCompactedBulkLoadedFiles,,1731767561887.8d5b640ddf71fb706e1914be3c6fed83. 2024-11-16T14:32:46,243 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testCompactedBulkLoadedFiles,,1731767561887.8d5b640ddf71fb706e1914be3c6fed83. after waiting 0 ms 2024-11-16T14:32:46,243 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testCompactedBulkLoadedFiles,,1731767561887.8d5b640ddf71fb706e1914be3c6fed83. 2024-11-16T14:32:46,244 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testCompactedBulkLoadedFiles,,1731767561887.8d5b640ddf71fb706e1914be3c6fed83. 2024-11-16T14:32:46,244 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 8d5b640ddf71fb706e1914be3c6fed83: Waiting for close lock at 1731767566243Disabling compacts and flushes for region at 1731767566243Disabling writes for close at 1731767566243Writing region close event to WAL at 1731767566244 (+1 ms)Closed at 1731767566244 2024-11-16T14:32:46,246 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33005 is added to blk_1073741924_1103 (size=95) 2024-11-16T14:32:46,246 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43279 is added to blk_1073741924_1103 (size=95) 2024-11-16T14:32:46,246 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36193 is added to blk_1073741924_1103 (size=95) 2024-11-16T14:32:46,248 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-16T14:32:46,248 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1731767566177) 2024-11-16T14:32:46,262 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayValueCompression#testCompactedBulkLoadedFiles Thread=442 (was 429) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-860658764_22 at /127.0.0.1:48346 [Waiting for operation #13] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-26-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1319272049) connection to localhost/127.0.0.1:43457 from jenkinstestCompactedBulkLoadedFiles java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: AsyncFSWAL-26-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-26-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkinstestCompactedBulkLoadedFiles@localhost:43457 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-860658764_22 at /127.0.0.1:54410 [Waiting for operation #3] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-860658764_22 at /127.0.0.1:40722 [Waiting for operation #3] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1346 (was 1264) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=163 (was 142) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=4461 (was 4442) - AvailableMemoryMB LEAK? - 2024-11-16T14:32:46,262 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1346 is superior to 1024 2024-11-16T14:32:46,275 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayValueCompression#testReplayEditsWrittenViaHRegion Thread=442, OpenFileDescriptor=1346, MaxFileDescriptor=1048576, SystemLoadAverage=163, ProcessCount=11, AvailableMemoryMB=4460 2024-11-16T14:32:46,275 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1346 is superior to 1024 2024-11-16T14:32:46,289 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-16T14:32:46,290 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-16T14:32:46,291 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-16T14:32:46,294 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-76451049, suffix=, logDir=hdfs://localhost:43457/hbase/WALs/hregion-76451049, archiveDir=hdfs://localhost:43457/hbase/oldWALs, maxLogs=32 2024-11-16T14:32:46,310 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-76451049/hregion-76451049.1731767566294, exclude list is [], retry=0 2024-11-16T14:32:46,313 DEBUG [AsyncFSWAL-28-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43279,DS-17ab9554-4509-49b3-918b-178692b9a473,DISK] 2024-11-16T14:32:46,313 DEBUG [AsyncFSWAL-28-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36193,DS-a30e140c-8532-4096-8cae-c8d713a3d4e3,DISK] 2024-11-16T14:32:46,313 DEBUG [AsyncFSWAL-28-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33005,DS-6a1dd528-c0be-4580-a53e-7113879f7f97,DISK] 2024-11-16T14:32:46,315 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-76451049/hregion-76451049.1731767566294 2024-11-16T14:32:46,315 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36015:36015),(127.0.0.1/127.0.0.1:35677:35677),(127.0.0.1/127.0.0.1:41179:41179)] 2024-11-16T14:32:46,315 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 4b2cd0a7fd3f295baf02bb7c13b447e9, NAME => 'testReplayEditsWrittenViaHRegion,,1731767566289.4b2cd0a7fd3f295baf02bb7c13b447e9.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenViaHRegion', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43457/hbase 2024-11-16T14:32:46,322 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36193 is added to blk_1073741926_1105 (size=67) 2024-11-16T14:32:46,322 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33005 is added to blk_1073741926_1105 (size=67) 2024-11-16T14:32:46,322 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43279 is added to blk_1073741926_1105 (size=67) 2024-11-16T14:32:46,323 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1731767566289.4b2cd0a7fd3f295baf02bb7c13b447e9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T14:32:46,324 INFO [StoreOpener-4b2cd0a7fd3f295baf02bb7c13b447e9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 4b2cd0a7fd3f295baf02bb7c13b447e9 2024-11-16T14:32:46,325 INFO [StoreOpener-4b2cd0a7fd3f295baf02bb7c13b447e9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4b2cd0a7fd3f295baf02bb7c13b447e9 columnFamilyName a 2024-11-16T14:32:46,325 DEBUG [StoreOpener-4b2cd0a7fd3f295baf02bb7c13b447e9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:32:46,326 INFO [StoreOpener-4b2cd0a7fd3f295baf02bb7c13b447e9-1 {}] regionserver.HStore(327): Store=4b2cd0a7fd3f295baf02bb7c13b447e9/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T14:32:46,326 INFO [StoreOpener-4b2cd0a7fd3f295baf02bb7c13b447e9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 4b2cd0a7fd3f295baf02bb7c13b447e9 2024-11-16T14:32:46,327 INFO [StoreOpener-4b2cd0a7fd3f295baf02bb7c13b447e9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4b2cd0a7fd3f295baf02bb7c13b447e9 columnFamilyName b 2024-11-16T14:32:46,327 DEBUG [StoreOpener-4b2cd0a7fd3f295baf02bb7c13b447e9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:32:46,327 INFO [StoreOpener-4b2cd0a7fd3f295baf02bb7c13b447e9-1 {}] regionserver.HStore(327): Store=4b2cd0a7fd3f295baf02bb7c13b447e9/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T14:32:46,328 INFO [StoreOpener-4b2cd0a7fd3f295baf02bb7c13b447e9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 4b2cd0a7fd3f295baf02bb7c13b447e9 2024-11-16T14:32:46,329 INFO [StoreOpener-4b2cd0a7fd3f295baf02bb7c13b447e9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4b2cd0a7fd3f295baf02bb7c13b447e9 columnFamilyName c 2024-11-16T14:32:46,329 DEBUG [StoreOpener-4b2cd0a7fd3f295baf02bb7c13b447e9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:32:46,329 INFO [StoreOpener-4b2cd0a7fd3f295baf02bb7c13b447e9-1 {}] regionserver.HStore(327): Store=4b2cd0a7fd3f295baf02bb7c13b447e9/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T14:32:46,330 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 4b2cd0a7fd3f295baf02bb7c13b447e9 2024-11-16T14:32:46,330 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43457/hbase/data/default/testReplayEditsWrittenViaHRegion/4b2cd0a7fd3f295baf02bb7c13b447e9 2024-11-16T14:32:46,331 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43457/hbase/data/default/testReplayEditsWrittenViaHRegion/4b2cd0a7fd3f295baf02bb7c13b447e9 2024-11-16T14:32:46,332 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 4b2cd0a7fd3f295baf02bb7c13b447e9 2024-11-16T14:32:46,332 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 4b2cd0a7fd3f295baf02bb7c13b447e9 2024-11-16T14:32:46,332 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-16T14:32:46,333 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 4b2cd0a7fd3f295baf02bb7c13b447e9 2024-11-16T14:32:46,335 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43457/hbase/data/default/testReplayEditsWrittenViaHRegion/4b2cd0a7fd3f295baf02bb7c13b447e9/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-16T14:32:46,336 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 4b2cd0a7fd3f295baf02bb7c13b447e9; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63045130, jitterRate=-0.06055435538291931}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-16T14:32:46,336 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 4b2cd0a7fd3f295baf02bb7c13b447e9: Writing region info on filesystem at 1731767566323Initializing all the Stores at 1731767566324 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731767566324Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731767566324Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731767566324Cleaning up temporary data from old regions at 1731767566332 (+8 ms)Region opened successfully at 1731767566336 (+4 ms) 2024-11-16T14:32:46,336 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 4b2cd0a7fd3f295baf02bb7c13b447e9, disabling compactions & flushes 2024-11-16T14:32:46,336 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1731767566289.4b2cd0a7fd3f295baf02bb7c13b447e9. 2024-11-16T14:32:46,336 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1731767566289.4b2cd0a7fd3f295baf02bb7c13b447e9. 2024-11-16T14:32:46,336 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1731767566289.4b2cd0a7fd3f295baf02bb7c13b447e9. after waiting 0 ms 2024-11-16T14:32:46,336 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1731767566289.4b2cd0a7fd3f295baf02bb7c13b447e9. 2024-11-16T14:32:46,337 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1731767566289.4b2cd0a7fd3f295baf02bb7c13b447e9. 2024-11-16T14:32:46,337 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 4b2cd0a7fd3f295baf02bb7c13b447e9: Waiting for close lock at 1731767566336Disabling compacts and flushes for region at 1731767566336Disabling writes for close at 1731767566336Writing region close event to WAL at 1731767566337 (+1 ms)Closed at 1731767566337 2024-11-16T14:32:46,339 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43279 is added to blk_1073741925_1104 (size=95) 2024-11-16T14:32:46,339 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36193 is added to blk_1073741925_1104 (size=95) 2024-11-16T14:32:46,339 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33005 is added to blk_1073741925_1104 (size=95) 2024-11-16T14:32:46,341 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-16T14:32:46,341 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-76451049:(num 1731767566294) 2024-11-16T14:32:46,341 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-16T14:32:46,343 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:43457/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731767566288, archiveDir=hdfs://localhost:43457/hbase/oldWALs, maxLogs=32 2024-11-16T14:32:46,357 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731767566288/wal.1731767566343, exclude list is [], retry=0 2024-11-16T14:32:46,359 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33005,DS-6a1dd528-c0be-4580-a53e-7113879f7f97,DISK] 2024-11-16T14:32:46,360 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43279,DS-17ab9554-4509-49b3-918b-178692b9a473,DISK] 2024-11-16T14:32:46,360 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36193,DS-a30e140c-8532-4096-8cae-c8d713a3d4e3,DISK] 2024-11-16T14:32:46,361 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731767566288/wal.1731767566343 2024-11-16T14:32:46,361 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41179:41179),(127.0.0.1/127.0.0.1:36015:36015),(127.0.0.1/127.0.0.1:35677:35677)] 2024-11-16T14:32:46,362 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 4b2cd0a7fd3f295baf02bb7c13b447e9, NAME => 'testReplayEditsWrittenViaHRegion,,1731767566289.4b2cd0a7fd3f295baf02bb7c13b447e9.', STARTKEY => '', ENDKEY => ''} 2024-11-16T14:32:46,362 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1731767566289.4b2cd0a7fd3f295baf02bb7c13b447e9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T14:32:46,362 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 4b2cd0a7fd3f295baf02bb7c13b447e9 2024-11-16T14:32:46,362 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 4b2cd0a7fd3f295baf02bb7c13b447e9 2024-11-16T14:32:46,363 INFO [StoreOpener-4b2cd0a7fd3f295baf02bb7c13b447e9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 4b2cd0a7fd3f295baf02bb7c13b447e9 2024-11-16T14:32:46,364 INFO [StoreOpener-4b2cd0a7fd3f295baf02bb7c13b447e9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4b2cd0a7fd3f295baf02bb7c13b447e9 columnFamilyName a 2024-11-16T14:32:46,364 DEBUG [StoreOpener-4b2cd0a7fd3f295baf02bb7c13b447e9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:32:46,365 INFO [StoreOpener-4b2cd0a7fd3f295baf02bb7c13b447e9-1 {}] regionserver.HStore(327): Store=4b2cd0a7fd3f295baf02bb7c13b447e9/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T14:32:46,365 INFO [StoreOpener-4b2cd0a7fd3f295baf02bb7c13b447e9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 4b2cd0a7fd3f295baf02bb7c13b447e9 2024-11-16T14:32:46,365 INFO [StoreOpener-4b2cd0a7fd3f295baf02bb7c13b447e9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4b2cd0a7fd3f295baf02bb7c13b447e9 columnFamilyName b 2024-11-16T14:32:46,365 DEBUG [StoreOpener-4b2cd0a7fd3f295baf02bb7c13b447e9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:32:46,366 INFO [StoreOpener-4b2cd0a7fd3f295baf02bb7c13b447e9-1 {}] regionserver.HStore(327): Store=4b2cd0a7fd3f295baf02bb7c13b447e9/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T14:32:46,366 INFO [StoreOpener-4b2cd0a7fd3f295baf02bb7c13b447e9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 4b2cd0a7fd3f295baf02bb7c13b447e9 2024-11-16T14:32:46,366 INFO [StoreOpener-4b2cd0a7fd3f295baf02bb7c13b447e9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4b2cd0a7fd3f295baf02bb7c13b447e9 columnFamilyName c 2024-11-16T14:32:46,367 DEBUG [StoreOpener-4b2cd0a7fd3f295baf02bb7c13b447e9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:32:46,367 INFO [StoreOpener-4b2cd0a7fd3f295baf02bb7c13b447e9-1 {}] regionserver.HStore(327): Store=4b2cd0a7fd3f295baf02bb7c13b447e9/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T14:32:46,367 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 4b2cd0a7fd3f295baf02bb7c13b447e9 2024-11-16T14:32:46,368 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43457/hbase/data/default/testReplayEditsWrittenViaHRegion/4b2cd0a7fd3f295baf02bb7c13b447e9 2024-11-16T14:32:46,369 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43457/hbase/data/default/testReplayEditsWrittenViaHRegion/4b2cd0a7fd3f295baf02bb7c13b447e9 2024-11-16T14:32:46,370 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 4b2cd0a7fd3f295baf02bb7c13b447e9 2024-11-16T14:32:46,370 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 4b2cd0a7fd3f295baf02bb7c13b447e9 2024-11-16T14:32:46,370 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-16T14:32:46,371 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 4b2cd0a7fd3f295baf02bb7c13b447e9 2024-11-16T14:32:46,372 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 4b2cd0a7fd3f295baf02bb7c13b447e9; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59776496, jitterRate=-0.10926079750061035}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-16T14:32:46,372 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 4b2cd0a7fd3f295baf02bb7c13b447e9: Writing region info on filesystem at 1731767566362Initializing all the Stores at 1731767566363 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731767566363Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731767566363Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731767566363Cleaning up temporary data from old regions at 1731767566370 (+7 ms)Region opened successfully at 1731767566372 (+2 ms) 2024-11-16T14:32:46,381 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 4b2cd0a7fd3f295baf02bb7c13b447e9 3/3 column families, dataSize=870 B heapSize=2.31 KB 2024-11-16T14:32:46,396 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43457/hbase/data/default/testReplayEditsWrittenViaHRegion/4b2cd0a7fd3f295baf02bb7c13b447e9/.tmp/a/7686187c3db14645b314a828cb375e6a is 91, key is testReplayEditsWrittenViaHRegion/a:x0/1731767566372/Put/seqid=0 2024-11-16T14:32:46,401 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33005 is added to blk_1073741928_1107 (size=5958) 2024-11-16T14:32:46,401 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36193 is added to blk_1073741928_1107 (size=5958) 2024-11-16T14:32:46,401 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43279 is added to blk_1073741928_1107 (size=5958) 2024-11-16T14:32:46,402 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:43457/hbase/data/default/testReplayEditsWrittenViaHRegion/4b2cd0a7fd3f295baf02bb7c13b447e9/.tmp/a/7686187c3db14645b314a828cb375e6a 2024-11-16T14:32:46,407 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43457/hbase/data/default/testReplayEditsWrittenViaHRegion/4b2cd0a7fd3f295baf02bb7c13b447e9/.tmp/a/7686187c3db14645b314a828cb375e6a as hdfs://localhost:43457/hbase/data/default/testReplayEditsWrittenViaHRegion/4b2cd0a7fd3f295baf02bb7c13b447e9/a/7686187c3db14645b314a828cb375e6a 2024-11-16T14:32:46,411 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43457/hbase/data/default/testReplayEditsWrittenViaHRegion/4b2cd0a7fd3f295baf02bb7c13b447e9/a/7686187c3db14645b314a828cb375e6a, entries=10, sequenceid=13, filesize=5.8 K 2024-11-16T14:32:46,412 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~870 B/870, heapSize ~1.80 KB/1840, currentSize=0 B/0 for 4b2cd0a7fd3f295baf02bb7c13b447e9 in 32ms, sequenceid=13, compaction requested=false 2024-11-16T14:32:46,412 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 4b2cd0a7fd3f295baf02bb7c13b447e9: 2024-11-16T14:32:46,429 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 4b2cd0a7fd3f295baf02bb7c13b447e9, disabling compactions & flushes 2024-11-16T14:32:46,429 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1731767566289.4b2cd0a7fd3f295baf02bb7c13b447e9. 2024-11-16T14:32:46,429 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1731767566289.4b2cd0a7fd3f295baf02bb7c13b447e9. 2024-11-16T14:32:46,429 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1731767566289.4b2cd0a7fd3f295baf02bb7c13b447e9. after waiting 0 ms 2024-11-16T14:32:46,429 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1731767566289.4b2cd0a7fd3f295baf02bb7c13b447e9. 2024-11-16T14:32:46,430 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 1740 in region testReplayEditsWrittenViaHRegion,,1731767566289.4b2cd0a7fd3f295baf02bb7c13b447e9. 2024-11-16T14:32:46,430 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1731767566289.4b2cd0a7fd3f295baf02bb7c13b447e9. 2024-11-16T14:32:46,430 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 4b2cd0a7fd3f295baf02bb7c13b447e9: Waiting for close lock at 1731767566429Disabling compacts and flushes for region at 1731767566429Disabling writes for close at 1731767566429Writing region close event to WAL at 1731767566430 (+1 ms)Closed at 1731767566430 2024-11-16T14:32:46,433 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36193 is added to blk_1073741927_1106 (size=3345) 2024-11-16T14:32:46,433 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43279 is added to blk_1073741927_1106 (size=3345) 2024-11-16T14:32:46,433 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33005 is added to blk_1073741927_1106 (size=3345) 2024-11-16T14:32:46,448 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:43457/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731767566288/wal.1731767566343, size=3.3 K (3345bytes) 2024-11-16T14:32:46,448 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:43457/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731767566288/wal.1731767566343 2024-11-16T14:32:46,448 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:43457/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731767566288/wal.1731767566343 after 0ms 2024-11-16T14:32:46,450 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:43457/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731767566288/wal.1731767566343: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-16T14:32:46,450 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:43457/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731767566288/wal.1731767566343 took 2ms 2024-11-16T14:32:46,452 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:43457/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731767566288/wal.1731767566343 so closing down 2024-11-16T14:32:46,452 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-16T14:32:46,453 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1731767566343.temp 2024-11-16T14:32:46,454 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:43457/hbase/data/default/testReplayEditsWrittenViaHRegion/4b2cd0a7fd3f295baf02bb7c13b447e9/recovered.edits/0000000000000000003-wal.1731767566343.temp 2024-11-16T14:32:46,454 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-16T14:32:46,460 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33005 is added to blk_1073741929_1108 (size=2944) 2024-11-16T14:32:46,460 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43279 is added to blk_1073741929_1108 (size=2944) 2024-11-16T14:32:46,460 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36193 is added to blk_1073741929_1108 (size=2944) 2024-11-16T14:32:46,460 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:43457/hbase/data/default/testReplayEditsWrittenViaHRegion/4b2cd0a7fd3f295baf02bb7c13b447e9/recovered.edits/0000000000000000003-wal.1731767566343.temp (wrote 30 edits, skipped 0 edits in 0 ms) 2024-11-16T14:32:46,462 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:43457/hbase/data/default/testReplayEditsWrittenViaHRegion/4b2cd0a7fd3f295baf02bb7c13b447e9/recovered.edits/0000000000000000003-wal.1731767566343.temp to hdfs://localhost:43457/hbase/data/default/testReplayEditsWrittenViaHRegion/4b2cd0a7fd3f295baf02bb7c13b447e9/recovered.edits/0000000000000000035 2024-11-16T14:32:46,462 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 32 edits across 1 Regions in 11 ms; skipped=2; WAL=hdfs://localhost:43457/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731767566288/wal.1731767566343, size=3.3 K, length=3345, corrupted=false, cancelled=false 2024-11-16T14:32:46,462 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:43457/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731767566288/wal.1731767566343, journal: Splitting hdfs://localhost:43457/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731767566288/wal.1731767566343, size=3.3 K (3345bytes) at 1731767566448Finishing writing output for hdfs://localhost:43457/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731767566288/wal.1731767566343 so closing down at 1731767566452 (+4 ms)Creating recovered edits writer path=hdfs://localhost:43457/hbase/data/default/testReplayEditsWrittenViaHRegion/4b2cd0a7fd3f295baf02bb7c13b447e9/recovered.edits/0000000000000000003-wal.1731767566343.temp at 1731767566454 (+2 ms)3 split writer threads finished at 1731767566454Closed recovered edits writer path=hdfs://localhost:43457/hbase/data/default/testReplayEditsWrittenViaHRegion/4b2cd0a7fd3f295baf02bb7c13b447e9/recovered.edits/0000000000000000003-wal.1731767566343.temp (wrote 30 edits, skipped 0 edits in 0 ms) at 1731767566460 (+6 ms)Rename recovered edits hdfs://localhost:43457/hbase/data/default/testReplayEditsWrittenViaHRegion/4b2cd0a7fd3f295baf02bb7c13b447e9/recovered.edits/0000000000000000003-wal.1731767566343.temp to hdfs://localhost:43457/hbase/data/default/testReplayEditsWrittenViaHRegion/4b2cd0a7fd3f295baf02bb7c13b447e9/recovered.edits/0000000000000000035 at 1731767566462 (+2 ms)Processed 32 edits across 1 Regions in 11 ms; skipped=2; WAL=hdfs://localhost:43457/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731767566288/wal.1731767566343, size=3.3 K, length=3345, corrupted=false, cancelled=false at 1731767566462 2024-11-16T14:32:46,463 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:43457/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731767566288/wal.1731767566343 to hdfs://localhost:43457/hbase/oldWALs/wal.1731767566343 2024-11-16T14:32:46,464 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:43457/hbase/data/default/testReplayEditsWrittenViaHRegion/4b2cd0a7fd3f295baf02bb7c13b447e9/recovered.edits/0000000000000000035 2024-11-16T14:32:46,464 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-16T14:32:46,466 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:43457/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731767566288, archiveDir=hdfs://localhost:43457/hbase/oldWALs, maxLogs=32 2024-11-16T14:32:46,478 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731767566288/wal.1731767566466, exclude list is [], retry=0 2024-11-16T14:32:46,481 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43279,DS-17ab9554-4509-49b3-918b-178692b9a473,DISK] 2024-11-16T14:32:46,481 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36193,DS-a30e140c-8532-4096-8cae-c8d713a3d4e3,DISK] 2024-11-16T14:32:46,481 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33005,DS-6a1dd528-c0be-4580-a53e-7113879f7f97,DISK] 2024-11-16T14:32:46,484 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731767566288/wal.1731767566466 2024-11-16T14:32:46,484 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36015:36015),(127.0.0.1/127.0.0.1:35677:35677),(127.0.0.1/127.0.0.1:41179:41179)] 2024-11-16T14:32:46,484 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 4b2cd0a7fd3f295baf02bb7c13b447e9, NAME => 'testReplayEditsWrittenViaHRegion,,1731767566289.4b2cd0a7fd3f295baf02bb7c13b447e9.', STARTKEY => '', ENDKEY => ''} 2024-11-16T14:32:46,484 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1731767566289.4b2cd0a7fd3f295baf02bb7c13b447e9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T14:32:46,484 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 4b2cd0a7fd3f295baf02bb7c13b447e9 2024-11-16T14:32:46,484 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 4b2cd0a7fd3f295baf02bb7c13b447e9 2024-11-16T14:32:46,486 INFO [StoreOpener-4b2cd0a7fd3f295baf02bb7c13b447e9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 4b2cd0a7fd3f295baf02bb7c13b447e9 2024-11-16T14:32:46,487 INFO [StoreOpener-4b2cd0a7fd3f295baf02bb7c13b447e9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4b2cd0a7fd3f295baf02bb7c13b447e9 columnFamilyName a 2024-11-16T14:32:46,487 DEBUG [StoreOpener-4b2cd0a7fd3f295baf02bb7c13b447e9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:32:46,492 DEBUG [StoreOpener-4b2cd0a7fd3f295baf02bb7c13b447e9-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:43457/hbase/data/default/testReplayEditsWrittenViaHRegion/4b2cd0a7fd3f295baf02bb7c13b447e9/a/7686187c3db14645b314a828cb375e6a 2024-11-16T14:32:46,492 INFO [StoreOpener-4b2cd0a7fd3f295baf02bb7c13b447e9-1 {}] regionserver.HStore(327): Store=4b2cd0a7fd3f295baf02bb7c13b447e9/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T14:32:46,493 INFO [StoreOpener-4b2cd0a7fd3f295baf02bb7c13b447e9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 4b2cd0a7fd3f295baf02bb7c13b447e9 2024-11-16T14:32:46,494 INFO [StoreOpener-4b2cd0a7fd3f295baf02bb7c13b447e9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4b2cd0a7fd3f295baf02bb7c13b447e9 columnFamilyName b 2024-11-16T14:32:46,494 DEBUG [StoreOpener-4b2cd0a7fd3f295baf02bb7c13b447e9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:32:46,494 INFO [StoreOpener-4b2cd0a7fd3f295baf02bb7c13b447e9-1 {}] regionserver.HStore(327): Store=4b2cd0a7fd3f295baf02bb7c13b447e9/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T14:32:46,494 INFO [StoreOpener-4b2cd0a7fd3f295baf02bb7c13b447e9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 4b2cd0a7fd3f295baf02bb7c13b447e9 2024-11-16T14:32:46,495 INFO [StoreOpener-4b2cd0a7fd3f295baf02bb7c13b447e9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4b2cd0a7fd3f295baf02bb7c13b447e9 columnFamilyName c 2024-11-16T14:32:46,495 DEBUG [StoreOpener-4b2cd0a7fd3f295baf02bb7c13b447e9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:32:46,495 INFO [StoreOpener-4b2cd0a7fd3f295baf02bb7c13b447e9-1 {}] regionserver.HStore(327): Store=4b2cd0a7fd3f295baf02bb7c13b447e9/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T14:32:46,496 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 4b2cd0a7fd3f295baf02bb7c13b447e9 2024-11-16T14:32:46,496 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43457/hbase/data/default/testReplayEditsWrittenViaHRegion/4b2cd0a7fd3f295baf02bb7c13b447e9 2024-11-16T14:32:46,498 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:43457/hbase/data/default/testReplayEditsWrittenViaHRegion/4b2cd0a7fd3f295baf02bb7c13b447e9 2024-11-16T14:32:46,498 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:43457/hbase/data/default/testReplayEditsWrittenViaHRegion/4b2cd0a7fd3f295baf02bb7c13b447e9/recovered.edits/0000000000000000035 2024-11-16T14:32:46,500 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:43457/hbase/data/default/testReplayEditsWrittenViaHRegion/4b2cd0a7fd3f295baf02bb7c13b447e9/recovered.edits/0000000000000000035: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-16T14:32:46,502 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 20, skipped 10, firstSequenceIdInLog=3, maxSequenceIdInLog=35, path=hdfs://localhost:43457/hbase/data/default/testReplayEditsWrittenViaHRegion/4b2cd0a7fd3f295baf02bb7c13b447e9/recovered.edits/0000000000000000035 2024-11-16T14:32:46,502 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 4b2cd0a7fd3f295baf02bb7c13b447e9 3/3 column families, dataSize=1.70 KB heapSize=3.88 KB 2024-11-16T14:32:46,515 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43457/hbase/data/default/testReplayEditsWrittenViaHRegion/4b2cd0a7fd3f295baf02bb7c13b447e9/.tmp/b/ae5418cbf16447668883190bddda608d is 91, key is testReplayEditsWrittenViaHRegion/b:x0/1731767566413/Put/seqid=0 2024-11-16T14:32:46,572 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33005 is added to blk_1073741931_1110 (size=5958) 2024-11-16T14:32:46,572 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43279 is added to blk_1073741931_1110 (size=5958) 2024-11-16T14:32:46,573 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-16T14:32:46,574 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36193 is added to blk_1073741931_1110 (size=5958) 2024-11-16T14:32:46,574 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=35 (bloomFilter=true), to=hdfs://localhost:43457/hbase/data/default/testReplayEditsWrittenViaHRegion/4b2cd0a7fd3f295baf02bb7c13b447e9/.tmp/b/ae5418cbf16447668883190bddda608d 2024-11-16T14:32:46,600 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43457/hbase/data/default/testReplayEditsWrittenViaHRegion/4b2cd0a7fd3f295baf02bb7c13b447e9/.tmp/c/828da4322ba64826a0ba9299bb717088 is 91, key is testReplayEditsWrittenViaHRegion/c:x0/1731767566419/Put/seqid=0 2024-11-16T14:32:46,608 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36193 is added to blk_1073741932_1111 (size=5958) 2024-11-16T14:32:46,608 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33005 is added to blk_1073741932_1111 (size=5958) 2024-11-16T14:32:46,608 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43279 is added to blk_1073741932_1111 (size=5958) 2024-11-16T14:32:46,608 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=35 (bloomFilter=true), to=hdfs://localhost:43457/hbase/data/default/testReplayEditsWrittenViaHRegion/4b2cd0a7fd3f295baf02bb7c13b447e9/.tmp/c/828da4322ba64826a0ba9299bb717088 2024-11-16T14:32:46,614 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43457/hbase/data/default/testReplayEditsWrittenViaHRegion/4b2cd0a7fd3f295baf02bb7c13b447e9/.tmp/b/ae5418cbf16447668883190bddda608d as hdfs://localhost:43457/hbase/data/default/testReplayEditsWrittenViaHRegion/4b2cd0a7fd3f295baf02bb7c13b447e9/b/ae5418cbf16447668883190bddda608d 2024-11-16T14:32:46,619 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43457/hbase/data/default/testReplayEditsWrittenViaHRegion/4b2cd0a7fd3f295baf02bb7c13b447e9/b/ae5418cbf16447668883190bddda608d, entries=10, sequenceid=35, filesize=5.8 K 2024-11-16T14:32:46,620 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43457/hbase/data/default/testReplayEditsWrittenViaHRegion/4b2cd0a7fd3f295baf02bb7c13b447e9/.tmp/c/828da4322ba64826a0ba9299bb717088 as hdfs://localhost:43457/hbase/data/default/testReplayEditsWrittenViaHRegion/4b2cd0a7fd3f295baf02bb7c13b447e9/c/828da4322ba64826a0ba9299bb717088 2024-11-16T14:32:46,625 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43457/hbase/data/default/testReplayEditsWrittenViaHRegion/4b2cd0a7fd3f295baf02bb7c13b447e9/c/828da4322ba64826a0ba9299bb717088, entries=10, sequenceid=35, filesize=5.8 K 2024-11-16T14:32:46,626 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~1.70 KB/1740, heapSize ~3.59 KB/3680, currentSize=0 B/0 for 4b2cd0a7fd3f295baf02bb7c13b447e9 in 124ms, sequenceid=35, compaction requested=false; wal=null 2024-11-16T14:32:46,626 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:43457/hbase/data/default/testReplayEditsWrittenViaHRegion/4b2cd0a7fd3f295baf02bb7c13b447e9/recovered.edits/0000000000000000035 2024-11-16T14:32:46,628 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 4b2cd0a7fd3f295baf02bb7c13b447e9 2024-11-16T14:32:46,628 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 4b2cd0a7fd3f295baf02bb7c13b447e9 2024-11-16T14:32:46,628 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-16T14:32:46,630 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 4b2cd0a7fd3f295baf02bb7c13b447e9 2024-11-16T14:32:46,632 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43457/hbase/data/default/testReplayEditsWrittenViaHRegion/4b2cd0a7fd3f295baf02bb7c13b447e9/recovered.edits/35.seqid, newMaxSeqId=35, maxSeqId=1 2024-11-16T14:32:46,633 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 4b2cd0a7fd3f295baf02bb7c13b447e9; next sequenceid=36; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66315537, jitterRate=-0.01182149350643158}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-16T14:32:46,633 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 4b2cd0a7fd3f295baf02bb7c13b447e9: Writing region info on filesystem at 1731767566484Initializing all the Stores at 1731767566485 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731767566485Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731767566485Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731767566485Obtaining lock to block concurrent updates at 1731767566502 (+17 ms)Preparing flush snapshotting stores in 4b2cd0a7fd3f295baf02bb7c13b447e9 at 1731767566502Finished memstore snapshotting testReplayEditsWrittenViaHRegion,,1731767566289.4b2cd0a7fd3f295baf02bb7c13b447e9., syncing WAL and waiting on mvcc, flushsize=dataSize=1740, getHeapSize=3920, getOffHeapSize=0, getCellsCount=20 at 1731767566502Flushing stores of testReplayEditsWrittenViaHRegion,,1731767566289.4b2cd0a7fd3f295baf02bb7c13b447e9. at 1731767566502Flushing 4b2cd0a7fd3f295baf02bb7c13b447e9/b: creating writer at 1731767566502Flushing 4b2cd0a7fd3f295baf02bb7c13b447e9/b: appending metadata at 1731767566514 (+12 ms)Flushing 4b2cd0a7fd3f295baf02bb7c13b447e9/b: closing flushed file at 1731767566514Flushing 4b2cd0a7fd3f295baf02bb7c13b447e9/c: creating writer at 1731767566580 (+66 ms)Flushing 4b2cd0a7fd3f295baf02bb7c13b447e9/c: appending metadata at 1731767566599 (+19 ms)Flushing 4b2cd0a7fd3f295baf02bb7c13b447e9/c: closing flushed file at 1731767566599Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@25ae9761: reopening flushed file at 1731767566613 (+14 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5fb443e3: reopening flushed file at 1731767566619 (+6 ms)Finished flush of dataSize ~1.70 KB/1740, heapSize ~3.59 KB/3680, currentSize=0 B/0 for 4b2cd0a7fd3f295baf02bb7c13b447e9 in 124ms, sequenceid=35, compaction requested=false; wal=null at 1731767566626 (+7 ms)Cleaning up temporary data from old regions at 1731767566628 (+2 ms)Region opened successfully at 1731767566633 (+5 ms) 2024-11-16T14:32:46,704 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:43457/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731767566288/wal.1731767566466, size=0 (0bytes) 2024-11-16T14:32:46,704 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:43457/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731767566288/wal.1731767566466 might be still open, length is 0 2024-11-16T14:32:46,704 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:43457/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731767566288/wal.1731767566466 2024-11-16T14:32:46,705 WARN [IPC Server handler 4 on default port 43457 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731767566288/wal.1731767566466 has not been closed. Lease recovery is in progress. RecoveryId = 1112 for block blk_1073741930_1109 2024-11-16T14:32:46,705 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:43457/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731767566288/wal.1731767566466 after 1ms 2024-11-16T14:32:47,261 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1995475656_22 at /127.0.0.1:48418 [Receiving block BP-865919460-172.17.0.2-1731767527819:blk_1073741930_1109] {}] datanode.DataXceiver(331): 127.0.0.1:43279:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:48418 dst: /127.0.0.1:43279 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:43279 remote=/127.0.0.1:48418]. Total timeout mills is 60000, 59405 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:32:47,262 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1995475656_22 at /127.0.0.1:54448 [Receiving block BP-865919460-172.17.0.2-1731767527819:blk_1073741930_1109] {}] datanode.DataXceiver(331): 127.0.0.1:33005:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:54448 dst: /127.0.0.1:33005 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:32:47,262 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1995475656_22 at /127.0.0.1:40770 [Receiving block BP-865919460-172.17.0.2-1731767527819:blk_1073741930_1109] {}] datanode.DataXceiver(331): 127.0.0.1:36193:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:40770 dst: /127.0.0.1:36193 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:32:47,266 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43279 is added to blk_1073741930_1112 (size=2936) 2024-11-16T14:32:47,267 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36193 is added to blk_1073741930_1112 (size=2936) 2024-11-16T14:32:50,707 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:43457/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731767566288/wal.1731767566466 after 4003ms 2024-11-16T14:32:50,712 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:43457/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731767566288/wal.1731767566466: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-16T14:32:50,712 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:43457/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731767566288/wal.1731767566466 took 4008ms 2024-11-16T14:32:50,715 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:43457/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731767566288/wal.1731767566466; continuing. 2024-11-16T14:32:50,715 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:43457/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731767566288/wal.1731767566466 so closing down 2024-11-16T14:32:50,715 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-16T14:32:50,716 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000037-wal.1731767566466.temp 2024-11-16T14:32:50,717 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:43457/hbase/data/default/testReplayEditsWrittenViaHRegion/4b2cd0a7fd3f295baf02bb7c13b447e9/recovered.edits/0000000000000000037-wal.1731767566466.temp 2024-11-16T14:32:50,718 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-16T14:32:50,723 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43279 is added to blk_1073741933_1113 (size=2944) 2024-11-16T14:32:50,723 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36193 is added to blk_1073741933_1113 (size=2944) 2024-11-16T14:32:50,724 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33005 is added to blk_1073741933_1113 (size=2944) 2024-11-16T14:32:50,724 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:43457/hbase/data/default/testReplayEditsWrittenViaHRegion/4b2cd0a7fd3f295baf02bb7c13b447e9/recovered.edits/0000000000000000037-wal.1731767566466.temp (wrote 30 edits, skipped 0 edits in 0 ms) 2024-11-16T14:32:50,726 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:43457/hbase/data/default/testReplayEditsWrittenViaHRegion/4b2cd0a7fd3f295baf02bb7c13b447e9/recovered.edits/0000000000000000037-wal.1731767566466.temp to hdfs://localhost:43457/hbase/data/default/testReplayEditsWrittenViaHRegion/4b2cd0a7fd3f295baf02bb7c13b447e9/recovered.edits/0000000000000000066 2024-11-16T14:32:50,726 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 30 edits across 1 Regions in 13 ms; skipped=0; WAL=hdfs://localhost:43457/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731767566288/wal.1731767566466, size=0, length=0, corrupted=false, cancelled=false 2024-11-16T14:32:50,726 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:43457/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731767566288/wal.1731767566466, journal: Splitting hdfs://localhost:43457/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731767566288/wal.1731767566466, size=0 (0bytes) at 1731767566704Finishing writing output for hdfs://localhost:43457/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731767566288/wal.1731767566466 so closing down at 1731767570715 (+4011 ms)Creating recovered edits writer path=hdfs://localhost:43457/hbase/data/default/testReplayEditsWrittenViaHRegion/4b2cd0a7fd3f295baf02bb7c13b447e9/recovered.edits/0000000000000000037-wal.1731767566466.temp at 1731767570717 (+2 ms)3 split writer threads finished at 1731767570718 (+1 ms)Closed recovered edits writer path=hdfs://localhost:43457/hbase/data/default/testReplayEditsWrittenViaHRegion/4b2cd0a7fd3f295baf02bb7c13b447e9/recovered.edits/0000000000000000037-wal.1731767566466.temp (wrote 30 edits, skipped 0 edits in 0 ms) at 1731767570724 (+6 ms)Rename recovered edits hdfs://localhost:43457/hbase/data/default/testReplayEditsWrittenViaHRegion/4b2cd0a7fd3f295baf02bb7c13b447e9/recovered.edits/0000000000000000037-wal.1731767566466.temp to hdfs://localhost:43457/hbase/data/default/testReplayEditsWrittenViaHRegion/4b2cd0a7fd3f295baf02bb7c13b447e9/recovered.edits/0000000000000000066 at 1731767570726 (+2 ms)Processed 30 edits across 1 Regions in 13 ms; skipped=0; WAL=hdfs://localhost:43457/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731767566288/wal.1731767566466, size=0, length=0, corrupted=false, cancelled=false at 1731767570726 2024-11-16T14:32:50,728 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:43457/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731767566288/wal.1731767566466 to hdfs://localhost:43457/hbase/oldWALs/wal.1731767566466 2024-11-16T14:32:50,729 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:43457/hbase/data/default/testReplayEditsWrittenViaHRegion/4b2cd0a7fd3f295baf02bb7c13b447e9/recovered.edits/0000000000000000066 2024-11-16T14:32:50,729 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-16T14:32:50,731 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:43457/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731767566288, archiveDir=hdfs://localhost:43457/hbase/oldWALs, maxLogs=32 2024-11-16T14:32:50,766 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731767566288/wal.1731767570732, exclude list is [], retry=0 2024-11-16T14:32:50,769 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33005,DS-6a1dd528-c0be-4580-a53e-7113879f7f97,DISK] 2024-11-16T14:32:50,769 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36193,DS-a30e140c-8532-4096-8cae-c8d713a3d4e3,DISK] 2024-11-16T14:32:50,769 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43279,DS-17ab9554-4509-49b3-918b-178692b9a473,DISK] 2024-11-16T14:32:50,771 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731767566288/wal.1731767570732 2024-11-16T14:32:50,771 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41179:41179),(127.0.0.1/127.0.0.1:35677:35677),(127.0.0.1/127.0.0.1:36015:36015)] 2024-11-16T14:32:50,772 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1731767566289.4b2cd0a7fd3f295baf02bb7c13b447e9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T14:32:50,773 INFO [StoreOpener-4b2cd0a7fd3f295baf02bb7c13b447e9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 4b2cd0a7fd3f295baf02bb7c13b447e9 2024-11-16T14:32:50,774 INFO [StoreOpener-4b2cd0a7fd3f295baf02bb7c13b447e9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4b2cd0a7fd3f295baf02bb7c13b447e9 columnFamilyName a 2024-11-16T14:32:50,774 DEBUG [StoreOpener-4b2cd0a7fd3f295baf02bb7c13b447e9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:32:50,778 DEBUG [StoreOpener-4b2cd0a7fd3f295baf02bb7c13b447e9-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:43457/hbase/data/default/testReplayEditsWrittenViaHRegion/4b2cd0a7fd3f295baf02bb7c13b447e9/a/7686187c3db14645b314a828cb375e6a 2024-11-16T14:32:50,778 INFO [StoreOpener-4b2cd0a7fd3f295baf02bb7c13b447e9-1 {}] regionserver.HStore(327): Store=4b2cd0a7fd3f295baf02bb7c13b447e9/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T14:32:50,778 INFO [StoreOpener-4b2cd0a7fd3f295baf02bb7c13b447e9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 4b2cd0a7fd3f295baf02bb7c13b447e9 2024-11-16T14:32:50,779 INFO [StoreOpener-4b2cd0a7fd3f295baf02bb7c13b447e9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4b2cd0a7fd3f295baf02bb7c13b447e9 columnFamilyName b 2024-11-16T14:32:50,779 DEBUG [StoreOpener-4b2cd0a7fd3f295baf02bb7c13b447e9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:32:50,783 DEBUG [StoreOpener-4b2cd0a7fd3f295baf02bb7c13b447e9-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:43457/hbase/data/default/testReplayEditsWrittenViaHRegion/4b2cd0a7fd3f295baf02bb7c13b447e9/b/ae5418cbf16447668883190bddda608d 2024-11-16T14:32:50,783 INFO [StoreOpener-4b2cd0a7fd3f295baf02bb7c13b447e9-1 {}] regionserver.HStore(327): Store=4b2cd0a7fd3f295baf02bb7c13b447e9/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T14:32:50,783 INFO [StoreOpener-4b2cd0a7fd3f295baf02bb7c13b447e9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 4b2cd0a7fd3f295baf02bb7c13b447e9 2024-11-16T14:32:50,784 INFO [StoreOpener-4b2cd0a7fd3f295baf02bb7c13b447e9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4b2cd0a7fd3f295baf02bb7c13b447e9 columnFamilyName c 2024-11-16T14:32:50,784 DEBUG [StoreOpener-4b2cd0a7fd3f295baf02bb7c13b447e9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:32:50,788 DEBUG [StoreOpener-4b2cd0a7fd3f295baf02bb7c13b447e9-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:43457/hbase/data/default/testReplayEditsWrittenViaHRegion/4b2cd0a7fd3f295baf02bb7c13b447e9/c/828da4322ba64826a0ba9299bb717088 2024-11-16T14:32:50,788 INFO [StoreOpener-4b2cd0a7fd3f295baf02bb7c13b447e9-1 {}] regionserver.HStore(327): Store=4b2cd0a7fd3f295baf02bb7c13b447e9/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T14:32:50,788 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 4b2cd0a7fd3f295baf02bb7c13b447e9 2024-11-16T14:32:50,789 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43457/hbase/data/default/testReplayEditsWrittenViaHRegion/4b2cd0a7fd3f295baf02bb7c13b447e9 2024-11-16T14:32:50,791 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:43457/hbase/data/default/testReplayEditsWrittenViaHRegion/4b2cd0a7fd3f295baf02bb7c13b447e9 2024-11-16T14:32:50,792 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:43457/hbase/data/default/testReplayEditsWrittenViaHRegion/4b2cd0a7fd3f295baf02bb7c13b447e9/recovered.edits/0000000000000000066 2024-11-16T14:32:50,794 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:43457/hbase/data/default/testReplayEditsWrittenViaHRegion/4b2cd0a7fd3f295baf02bb7c13b447e9/recovered.edits/0000000000000000066: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-16T14:32:50,798 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 30, skipped 0, firstSequenceIdInLog=37, maxSequenceIdInLog=66, path=hdfs://localhost:43457/hbase/data/default/testReplayEditsWrittenViaHRegion/4b2cd0a7fd3f295baf02bb7c13b447e9/recovered.edits/0000000000000000066 2024-11-16T14:32:50,798 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 4b2cd0a7fd3f295baf02bb7c13b447e9 3/3 column families, dataSize=2.55 KB heapSize=5.44 KB 2024-11-16T14:32:50,812 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43457/hbase/data/default/testReplayEditsWrittenViaHRegion/4b2cd0a7fd3f295baf02bb7c13b447e9/.tmp/a/8e57875415cc43b6a64d941a49bcbfa2 is 91, key is testReplayEditsWrittenViaHRegion/a:y0/1731767566642/Put/seqid=0 2024-11-16T14:32:50,817 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36193 is added to blk_1073741935_1115 (size=5958) 2024-11-16T14:32:50,817 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43279 is added to blk_1073741935_1115 (size=5958) 2024-11-16T14:32:50,818 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33005 is added to blk_1073741935_1115 (size=5958) 2024-11-16T14:32:50,818 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:43457/hbase/data/default/testReplayEditsWrittenViaHRegion/4b2cd0a7fd3f295baf02bb7c13b447e9/.tmp/a/8e57875415cc43b6a64d941a49bcbfa2 2024-11-16T14:32:50,834 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43457/hbase/data/default/testReplayEditsWrittenViaHRegion/4b2cd0a7fd3f295baf02bb7c13b447e9/.tmp/b/7684b962b56a41f697ec80041ffd853e is 91, key is testReplayEditsWrittenViaHRegion/b:y0/1731767566651/Put/seqid=0 2024-11-16T14:32:50,839 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36193 is added to blk_1073741936_1116 (size=5958) 2024-11-16T14:32:50,839 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43279 is added to blk_1073741936_1116 (size=5958) 2024-11-16T14:32:50,840 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33005 is added to blk_1073741936_1116 (size=5958) 2024-11-16T14:32:50,840 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:43457/hbase/data/default/testReplayEditsWrittenViaHRegion/4b2cd0a7fd3f295baf02bb7c13b447e9/.tmp/b/7684b962b56a41f697ec80041ffd853e 2024-11-16T14:32:50,861 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43457/hbase/data/default/testReplayEditsWrittenViaHRegion/4b2cd0a7fd3f295baf02bb7c13b447e9/.tmp/c/80284827273842ce9f7203d4355cea3b is 91, key is testReplayEditsWrittenViaHRegion/c:y0/1731767566659/Put/seqid=0 2024-11-16T14:32:50,866 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33005 is added to blk_1073741937_1117 (size=5958) 2024-11-16T14:32:50,866 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43279 is added to blk_1073741937_1117 (size=5958) 2024-11-16T14:32:50,867 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36193 is added to blk_1073741937_1117 (size=5958) 2024-11-16T14:32:50,867 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:43457/hbase/data/default/testReplayEditsWrittenViaHRegion/4b2cd0a7fd3f295baf02bb7c13b447e9/.tmp/c/80284827273842ce9f7203d4355cea3b 2024-11-16T14:32:50,872 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43457/hbase/data/default/testReplayEditsWrittenViaHRegion/4b2cd0a7fd3f295baf02bb7c13b447e9/.tmp/a/8e57875415cc43b6a64d941a49bcbfa2 as hdfs://localhost:43457/hbase/data/default/testReplayEditsWrittenViaHRegion/4b2cd0a7fd3f295baf02bb7c13b447e9/a/8e57875415cc43b6a64d941a49bcbfa2 2024-11-16T14:32:50,876 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43457/hbase/data/default/testReplayEditsWrittenViaHRegion/4b2cd0a7fd3f295baf02bb7c13b447e9/a/8e57875415cc43b6a64d941a49bcbfa2, entries=10, sequenceid=66, filesize=5.8 K 2024-11-16T14:32:50,877 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43457/hbase/data/default/testReplayEditsWrittenViaHRegion/4b2cd0a7fd3f295baf02bb7c13b447e9/.tmp/b/7684b962b56a41f697ec80041ffd853e as hdfs://localhost:43457/hbase/data/default/testReplayEditsWrittenViaHRegion/4b2cd0a7fd3f295baf02bb7c13b447e9/b/7684b962b56a41f697ec80041ffd853e 2024-11-16T14:32:50,882 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43457/hbase/data/default/testReplayEditsWrittenViaHRegion/4b2cd0a7fd3f295baf02bb7c13b447e9/b/7684b962b56a41f697ec80041ffd853e, entries=10, sequenceid=66, filesize=5.8 K 2024-11-16T14:32:50,883 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43457/hbase/data/default/testReplayEditsWrittenViaHRegion/4b2cd0a7fd3f295baf02bb7c13b447e9/.tmp/c/80284827273842ce9f7203d4355cea3b as hdfs://localhost:43457/hbase/data/default/testReplayEditsWrittenViaHRegion/4b2cd0a7fd3f295baf02bb7c13b447e9/c/80284827273842ce9f7203d4355cea3b 2024-11-16T14:32:50,887 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43457/hbase/data/default/testReplayEditsWrittenViaHRegion/4b2cd0a7fd3f295baf02bb7c13b447e9/c/80284827273842ce9f7203d4355cea3b, entries=10, sequenceid=66, filesize=5.8 K 2024-11-16T14:32:50,887 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~2.55 KB/2610, heapSize ~5.39 KB/5520, currentSize=0 B/0 for 4b2cd0a7fd3f295baf02bb7c13b447e9 in 89ms, sequenceid=66, compaction requested=false; wal=null 2024-11-16T14:32:50,888 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:43457/hbase/data/default/testReplayEditsWrittenViaHRegion/4b2cd0a7fd3f295baf02bb7c13b447e9/recovered.edits/0000000000000000066 2024-11-16T14:32:50,889 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 4b2cd0a7fd3f295baf02bb7c13b447e9 2024-11-16T14:32:50,889 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 4b2cd0a7fd3f295baf02bb7c13b447e9 2024-11-16T14:32:50,890 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-16T14:32:50,891 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 4b2cd0a7fd3f295baf02bb7c13b447e9 2024-11-16T14:32:50,893 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43457/hbase/data/default/testReplayEditsWrittenViaHRegion/4b2cd0a7fd3f295baf02bb7c13b447e9/recovered.edits/66.seqid, newMaxSeqId=66, maxSeqId=35 2024-11-16T14:32:50,894 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 4b2cd0a7fd3f295baf02bb7c13b447e9; next sequenceid=67; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71904138, jitterRate=0.07145515084266663}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-16T14:32:50,894 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 4b2cd0a7fd3f295baf02bb7c13b447e9: Writing region info on filesystem at 1731767570772Initializing all the Stores at 1731767570773 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731767570773Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731767570773Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731767570773Obtaining lock to block concurrent updates at 1731767570798 (+25 ms)Preparing flush snapshotting stores in 4b2cd0a7fd3f295baf02bb7c13b447e9 at 1731767570798Finished memstore snapshotting testReplayEditsWrittenViaHRegion,,1731767566289.4b2cd0a7fd3f295baf02bb7c13b447e9., syncing WAL and waiting on mvcc, flushsize=dataSize=2610, getHeapSize=5520, getOffHeapSize=0, getCellsCount=30 at 1731767570798Flushing stores of testReplayEditsWrittenViaHRegion,,1731767566289.4b2cd0a7fd3f295baf02bb7c13b447e9. at 1731767570798Flushing 4b2cd0a7fd3f295baf02bb7c13b447e9/a: creating writer at 1731767570798Flushing 4b2cd0a7fd3f295baf02bb7c13b447e9/a: appending metadata at 1731767570812 (+14 ms)Flushing 4b2cd0a7fd3f295baf02bb7c13b447e9/a: closing flushed file at 1731767570812Flushing 4b2cd0a7fd3f295baf02bb7c13b447e9/b: creating writer at 1731767570822 (+10 ms)Flushing 4b2cd0a7fd3f295baf02bb7c13b447e9/b: appending metadata at 1731767570834 (+12 ms)Flushing 4b2cd0a7fd3f295baf02bb7c13b447e9/b: closing flushed file at 1731767570834Flushing 4b2cd0a7fd3f295baf02bb7c13b447e9/c: creating writer at 1731767570844 (+10 ms)Flushing 4b2cd0a7fd3f295baf02bb7c13b447e9/c: appending metadata at 1731767570860 (+16 ms)Flushing 4b2cd0a7fd3f295baf02bb7c13b447e9/c: closing flushed file at 1731767570860Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1b31e36: reopening flushed file at 1731767570871 (+11 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@35637a74: reopening flushed file at 1731767570876 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@498cb77: reopening flushed file at 1731767570882 (+6 ms)Finished flush of dataSize ~2.55 KB/2610, heapSize ~5.39 KB/5520, currentSize=0 B/0 for 4b2cd0a7fd3f295baf02bb7c13b447e9 in 89ms, sequenceid=66, compaction requested=false; wal=null at 1731767570887 (+5 ms)Cleaning up temporary data from old regions at 1731767570889 (+2 ms)Region opened successfully at 1731767570894 (+5 ms) 2024-11-16T14:32:50,908 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 4b2cd0a7fd3f295baf02bb7c13b447e9, disabling compactions & flushes 2024-11-16T14:32:50,908 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1731767566289.4b2cd0a7fd3f295baf02bb7c13b447e9. 2024-11-16T14:32:50,908 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1731767566289.4b2cd0a7fd3f295baf02bb7c13b447e9. 2024-11-16T14:32:50,908 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1731767566289.4b2cd0a7fd3f295baf02bb7c13b447e9. after waiting 0 ms 2024-11-16T14:32:50,908 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1731767566289.4b2cd0a7fd3f295baf02bb7c13b447e9. 2024-11-16T14:32:50,910 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1731767566289.4b2cd0a7fd3f295baf02bb7c13b447e9. 2024-11-16T14:32:50,910 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 4b2cd0a7fd3f295baf02bb7c13b447e9: Waiting for close lock at 1731767570908Disabling compacts and flushes for region at 1731767570908Disabling writes for close at 1731767570908Writing region close event to WAL at 1731767570910 (+2 ms)Closed at 1731767570910 2024-11-16T14:32:50,913 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43279 is added to blk_1073741934_1114 (size=95) 2024-11-16T14:32:50,913 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33005 is added to blk_1073741934_1114 (size=95) 2024-11-16T14:32:50,913 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36193 is added to blk_1073741934_1114 (size=95) 2024-11-16T14:32:50,915 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-16T14:32:50,915 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1731767570732) 2024-11-16T14:32:50,929 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayValueCompression#testReplayEditsWrittenViaHRegion Thread=446 (was 442) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1871322878_22 at /127.0.0.1:40830 [Waiting for operation #3] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1871322878_22 at /127.0.0.1:54482 [Waiting for operation #27] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1871322878_22 at /127.0.0.1:48488 [Waiting for operation #13] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-28-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-28-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1319272049) connection to localhost/127.0.0.1:43457 from jenkinstestReplayEditsWrittenViaHRegion java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: AsyncFSWAL-28-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkinstestReplayEditsWrittenViaHRegion@localhost:43457 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1412 (was 1346) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=150 (was 163), ProcessCount=11 (was 11), AvailableMemoryMB=4426 (was 4460) 2024-11-16T14:32:50,929 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1412 is superior to 1024 2024-11-16T14:32:50,929 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-16T14:32:50,929 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-16T14:32:50,929 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-16T14:32:50,929 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:32:50,930 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:32:50,930 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T14:32:50,930 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-16T14:32:50,930 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=387325396, stopped=false 2024-11-16T14:32:50,930 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=04e93f2b5fad,34365,1731767532092 2024-11-16T14:32:50,989 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39921-0x101440f72da0002, quorum=127.0.0.1:57212, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-16T14:32:50,989 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35241-0x101440f72da0001, quorum=127.0.0.1:57212, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-16T14:32:50,989 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34365-0x101440f72da0000, quorum=127.0.0.1:57212, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-16T14:32:50,989 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34365-0x101440f72da0000, quorum=127.0.0.1:57212, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T14:32:50,989 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39921-0x101440f72da0002, quorum=127.0.0.1:57212, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T14:32:50,989 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-16T14:32:50,989 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35241-0x101440f72da0001, quorum=127.0.0.1:57212, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T14:32:50,990 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:34365-0x101440f72da0000, quorum=127.0.0.1:57212, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-16T14:32:50,990 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:35241-0x101440f72da0001, quorum=127.0.0.1:57212, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-16T14:32:50,990 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:39921-0x101440f72da0002, quorum=127.0.0.1:57212, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-16T14:32:50,990 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-16T14:32:50,990 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-16T14:32:50,990 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:32:50,991 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '04e93f2b5fad,35241,1731767532873' ***** 2024-11-16T14:32:50,991 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-16T14:32:50,991 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '04e93f2b5fad,39921,1731767532970' ***** 2024-11-16T14:32:50,991 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-16T14:32:50,991 INFO [RS:1;04e93f2b5fad:39921 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-16T14:32:50,991 INFO [RS:0;04e93f2b5fad:35241 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-16T14:32:50,991 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-16T14:32:50,991 INFO [RS:1;04e93f2b5fad:39921 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-16T14:32:50,991 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-16T14:32:50,991 INFO [RS:0;04e93f2b5fad:35241 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-16T14:32:50,991 INFO [RS:1;04e93f2b5fad:39921 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-16T14:32:50,991 INFO [RS:0;04e93f2b5fad:35241 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-16T14:32:50,991 INFO [RS:1;04e93f2b5fad:39921 {}] regionserver.HRegionServer(3091): Received CLOSE for 9cc76f2c81b25eccf90da7ca97d0721e 2024-11-16T14:32:50,991 INFO [RS:0;04e93f2b5fad:35241 {}] regionserver.HRegionServer(959): stopping server 04e93f2b5fad,35241,1731767532873 2024-11-16T14:32:50,991 INFO [RS:0;04e93f2b5fad:35241 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-16T14:32:50,992 INFO [RS:0;04e93f2b5fad:35241 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;04e93f2b5fad:35241. 2024-11-16T14:32:50,992 DEBUG [RS:0;04e93f2b5fad:35241 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-16T14:32:50,992 INFO [RS:1;04e93f2b5fad:39921 {}] regionserver.HRegionServer(959): stopping server 04e93f2b5fad,39921,1731767532970 2024-11-16T14:32:50,992 DEBUG [RS:0;04e93f2b5fad:35241 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:32:50,992 INFO [RS:1;04e93f2b5fad:39921 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-16T14:32:50,992 INFO [RS:1;04e93f2b5fad:39921 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;04e93f2b5fad:39921. 2024-11-16T14:32:50,992 INFO [RS:0;04e93f2b5fad:35241 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-16T14:32:50,992 DEBUG [RS:1;04e93f2b5fad:39921 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-16T14:32:50,992 DEBUG [RS:1;04e93f2b5fad:39921 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:32:50,992 DEBUG [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 9cc76f2c81b25eccf90da7ca97d0721e, disabling compactions & flushes 2024-11-16T14:32:50,992 INFO [RS:1;04e93f2b5fad:39921 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-16T14:32:50,992 INFO [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1731767547738.9cc76f2c81b25eccf90da7ca97d0721e. 2024-11-16T14:32:50,992 DEBUG [RS:1;04e93f2b5fad:39921 {}] regionserver.HRegionServer(1325): Online Regions={9cc76f2c81b25eccf90da7ca97d0721e=testReplayEditsAfterRegionMovedWithMultiCF,,1731767547738.9cc76f2c81b25eccf90da7ca97d0721e.} 2024-11-16T14:32:50,992 DEBUG [RS:1;04e93f2b5fad:39921 {}] regionserver.HRegionServer(1351): Waiting on 9cc76f2c81b25eccf90da7ca97d0721e 2024-11-16T14:32:50,992 DEBUG [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731767547738.9cc76f2c81b25eccf90da7ca97d0721e. 2024-11-16T14:32:50,992 DEBUG [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731767547738.9cc76f2c81b25eccf90da7ca97d0721e. after waiting 0 ms 2024-11-16T14:32:50,992 DEBUG [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1731767547738.9cc76f2c81b25eccf90da7ca97d0721e. 2024-11-16T14:32:50,992 INFO [RS:0;04e93f2b5fad:35241 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-16T14:32:50,992 INFO [RS:0;04e93f2b5fad:35241 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-16T14:32:50,992 INFO [RS:0;04e93f2b5fad:35241 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-16T14:32:50,995 INFO [RS:0;04e93f2b5fad:35241 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-16T14:32:50,995 DEBUG [RS:0;04e93f2b5fad:35241 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-16T14:32:50,995 DEBUG [RS_CLOSE_META-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-16T14:32:50,995 DEBUG [RS:0;04e93f2b5fad:35241 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-16T14:32:50,995 INFO [RS_CLOSE_META-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-16T14:32:50,995 DEBUG [RS_CLOSE_META-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-16T14:32:50,995 DEBUG [RS_CLOSE_META-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-16T14:32:50,995 DEBUG [RS_CLOSE_META-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-16T14:32:50,995 INFO [RS_CLOSE_META-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=6.86 KB heapSize=11.45 KB 2024-11-16T14:32:50,996 INFO [regionserver/04e93f2b5fad:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-16T14:32:50,996 INFO [regionserver/04e93f2b5fad:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-16T14:32:51,000 DEBUG [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43457/user/jenkins/test-data/0d13f5ec-97ce-91d4-7824-c5174956db56/data/default/testReplayEditsAfterRegionMovedWithMultiCF/9cc76f2c81b25eccf90da7ca97d0721e/recovered.edits/20.seqid, newMaxSeqId=20, maxSeqId=17 2024-11-16T14:32:51,001 INFO [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1731767547738.9cc76f2c81b25eccf90da7ca97d0721e. 2024-11-16T14:32:51,001 DEBUG [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 9cc76f2c81b25eccf90da7ca97d0721e: Waiting for close lock at 1731767570992Running coprocessor pre-close hooks at 1731767570992Disabling compacts and flushes for region at 1731767570992Disabling writes for close at 1731767570992Writing region close event to WAL at 1731767570996 (+4 ms)Running coprocessor post-close hooks at 1731767571001 (+5 ms)Closed at 1731767571001 2024-11-16T14:32:51,001 DEBUG [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1731767547738.9cc76f2c81b25eccf90da7ca97d0721e. 2024-11-16T14:32:51,016 DEBUG [RS_CLOSE_META-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43457/user/jenkins/test-data/0d13f5ec-97ce-91d4-7824-c5174956db56/data/hbase/meta/1588230740/.tmp/info/aef144dea44b413f900a873ea662ba91 is 205, key is testReplayEditsAfterRegionMovedWithMultiCF,,1731767547738.9cc76f2c81b25eccf90da7ca97d0721e./info:regioninfo/1731767550900/Put/seqid=0 2024-11-16T14:32:51,021 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33005 is added to blk_1073741938_1118 (size=8243) 2024-11-16T14:32:51,021 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36193 is added to blk_1073741938_1118 (size=8243) 2024-11-16T14:32:51,022 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43279 is added to blk_1073741938_1118 (size=8243) 2024-11-16T14:32:51,022 INFO [RS_CLOSE_META-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.65 KB at sequenceid=21 (bloomFilter=true), to=hdfs://localhost:43457/user/jenkins/test-data/0d13f5ec-97ce-91d4-7824-c5174956db56/data/hbase/meta/1588230740/.tmp/info/aef144dea44b413f900a873ea662ba91 2024-11-16T14:32:51,044 DEBUG [RS_CLOSE_META-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43457/user/jenkins/test-data/0d13f5ec-97ce-91d4-7824-c5174956db56/data/hbase/meta/1588230740/.tmp/ns/b1feb8e354d84208bddecf35b2f75f88 is 43, key is default/ns:d/1731767535162/Put/seqid=0 2024-11-16T14:32:51,049 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43279 is added to blk_1073741939_1119 (size=5153) 2024-11-16T14:32:51,049 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36193 is added to blk_1073741939_1119 (size=5153) 2024-11-16T14:32:51,050 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33005 is added to blk_1073741939_1119 (size=5153) 2024-11-16T14:32:51,050 INFO [RS_CLOSE_META-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=21 (bloomFilter=true), to=hdfs://localhost:43457/user/jenkins/test-data/0d13f5ec-97ce-91d4-7824-c5174956db56/data/hbase/meta/1588230740/.tmp/ns/b1feb8e354d84208bddecf35b2f75f88 2024-11-16T14:32:51,069 DEBUG [RS_CLOSE_META-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43457/user/jenkins/test-data/0d13f5ec-97ce-91d4-7824-c5174956db56/data/hbase/meta/1588230740/.tmp/table/49a861d809664107ad00464818670c19 is 78, key is testReplayEditsAfterRegionMovedWithMultiCF/table:state/1731767548178/Put/seqid=0 2024-11-16T14:32:51,074 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43279 is added to blk_1073741940_1120 (size=5431) 2024-11-16T14:32:51,075 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36193 is added to blk_1073741940_1120 (size=5431) 2024-11-16T14:32:51,075 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33005 is added to blk_1073741940_1120 (size=5431) 2024-11-16T14:32:51,076 INFO [RS_CLOSE_META-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=148 B at sequenceid=21 (bloomFilter=true), to=hdfs://localhost:43457/user/jenkins/test-data/0d13f5ec-97ce-91d4-7824-c5174956db56/data/hbase/meta/1588230740/.tmp/table/49a861d809664107ad00464818670c19 2024-11-16T14:32:51,081 DEBUG [RS_CLOSE_META-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43457/user/jenkins/test-data/0d13f5ec-97ce-91d4-7824-c5174956db56/data/hbase/meta/1588230740/.tmp/info/aef144dea44b413f900a873ea662ba91 as hdfs://localhost:43457/user/jenkins/test-data/0d13f5ec-97ce-91d4-7824-c5174956db56/data/hbase/meta/1588230740/info/aef144dea44b413f900a873ea662ba91 2024-11-16T14:32:51,087 INFO [RS_CLOSE_META-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43457/user/jenkins/test-data/0d13f5ec-97ce-91d4-7824-c5174956db56/data/hbase/meta/1588230740/info/aef144dea44b413f900a873ea662ba91, entries=18, sequenceid=21, filesize=8.0 K 2024-11-16T14:32:51,088 DEBUG [RS_CLOSE_META-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43457/user/jenkins/test-data/0d13f5ec-97ce-91d4-7824-c5174956db56/data/hbase/meta/1588230740/.tmp/ns/b1feb8e354d84208bddecf35b2f75f88 as hdfs://localhost:43457/user/jenkins/test-data/0d13f5ec-97ce-91d4-7824-c5174956db56/data/hbase/meta/1588230740/ns/b1feb8e354d84208bddecf35b2f75f88 2024-11-16T14:32:51,093 INFO [RS_CLOSE_META-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43457/user/jenkins/test-data/0d13f5ec-97ce-91d4-7824-c5174956db56/data/hbase/meta/1588230740/ns/b1feb8e354d84208bddecf35b2f75f88, entries=2, sequenceid=21, filesize=5.0 K 2024-11-16T14:32:51,094 DEBUG [RS_CLOSE_META-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43457/user/jenkins/test-data/0d13f5ec-97ce-91d4-7824-c5174956db56/data/hbase/meta/1588230740/.tmp/table/49a861d809664107ad00464818670c19 as hdfs://localhost:43457/user/jenkins/test-data/0d13f5ec-97ce-91d4-7824-c5174956db56/data/hbase/meta/1588230740/table/49a861d809664107ad00464818670c19 2024-11-16T14:32:51,099 INFO [RS_CLOSE_META-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43457/user/jenkins/test-data/0d13f5ec-97ce-91d4-7824-c5174956db56/data/hbase/meta/1588230740/table/49a861d809664107ad00464818670c19, entries=2, sequenceid=21, filesize=5.3 K 2024-11-16T14:32:51,100 INFO [RS_CLOSE_META-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~6.86 KB/7029, heapSize ~11.16 KB/11424, currentSize=0 B/0 for 1588230740 in 105ms, sequenceid=21, compaction requested=false 2024-11-16T14:32:51,105 DEBUG [RS_CLOSE_META-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43457/user/jenkins/test-data/0d13f5ec-97ce-91d4-7824-c5174956db56/data/hbase/meta/1588230740/recovered.edits/24.seqid, newMaxSeqId=24, maxSeqId=1 2024-11-16T14:32:51,105 DEBUG [RS_CLOSE_META-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-16T14:32:51,105 INFO [RS_CLOSE_META-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-16T14:32:51,105 DEBUG [RS_CLOSE_META-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731767570995Running coprocessor pre-close hooks at 1731767570995Disabling compacts and flushes for region at 1731767570995Disabling writes for close at 1731767570995Obtaining lock to block concurrent updates at 1731767570995Preparing flush snapshotting stores in 1588230740 at 1731767570995Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=7029, getHeapSize=11664, getOffHeapSize=0, getCellsCount=48 at 1731767570996 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1731767570996Flushing 1588230740/info: creating writer at 1731767570997 (+1 ms)Flushing 1588230740/info: appending metadata at 1731767571015 (+18 ms)Flushing 1588230740/info: closing flushed file at 1731767571015Flushing 1588230740/ns: creating writer at 1731767571028 (+13 ms)Flushing 1588230740/ns: appending metadata at 1731767571043 (+15 ms)Flushing 1588230740/ns: closing flushed file at 1731767571043Flushing 1588230740/table: creating writer at 1731767571054 (+11 ms)Flushing 1588230740/table: appending metadata at 1731767571068 (+14 ms)Flushing 1588230740/table: closing flushed file at 1731767571068Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@50042afc: reopening flushed file at 1731767571081 (+13 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@249e7619: reopening flushed file at 1731767571087 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4332af7d: reopening flushed file at 1731767571094 (+7 ms)Finished flush of dataSize ~6.86 KB/7029, heapSize ~11.16 KB/11424, currentSize=0 B/0 for 1588230740 in 105ms, sequenceid=21, compaction requested=false at 1731767571100 (+6 ms)Writing region close event to WAL at 1731767571102 (+2 ms)Running coprocessor post-close hooks at 1731767571105 (+3 ms)Closed at 1731767571105 2024-11-16T14:32:51,105 DEBUG [RS_CLOSE_META-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-16T14:32:51,192 INFO [RS:1;04e93f2b5fad:39921 {}] regionserver.HRegionServer(976): stopping server 04e93f2b5fad,39921,1731767532970; all regions closed. 2024-11-16T14:32:51,195 INFO [RS:0;04e93f2b5fad:35241 {}] regionserver.HRegionServer(976): stopping server 04e93f2b5fad,35241,1731767532873; all regions closed. 2024-11-16T14:32:51,198 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43279 is added to blk_1073741835_1011 (size=720) 2024-11-16T14:32:51,198 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33005 is added to blk_1073741835_1011 (size=720) 2024-11-16T14:32:51,198 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36193 is added to blk_1073741835_1011 (size=720) 2024-11-16T14:32:51,201 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36193 is added to blk_1073741836_1012 (size=4717) 2024-11-16T14:32:51,201 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33005 is added to blk_1073741836_1012 (size=4717) 2024-11-16T14:32:51,201 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43279 is added to blk_1073741836_1012 (size=4717) 2024-11-16T14:32:51,203 DEBUG [RS:1;04e93f2b5fad:39921 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/0d13f5ec-97ce-91d4-7824-c5174956db56/oldWALs 2024-11-16T14:32:51,203 INFO [RS:1;04e93f2b5fad:39921 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 04e93f2b5fad%2C39921%2C1731767532970:(num 1731767534512) 2024-11-16T14:32:51,203 DEBUG [RS:1;04e93f2b5fad:39921 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:32:51,203 INFO [RS:1;04e93f2b5fad:39921 {}] regionserver.LeaseManager(133): Closed leases 2024-11-16T14:32:51,203 INFO [RS:1;04e93f2b5fad:39921 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-16T14:32:51,203 INFO [RS:1;04e93f2b5fad:39921 {}] hbase.ChoreService(370): Chore service for: regionserver/04e93f2b5fad:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-16T14:32:51,203 INFO [RS:1;04e93f2b5fad:39921 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-16T14:32:51,203 INFO [regionserver/04e93f2b5fad:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-16T14:32:51,203 INFO [RS:1;04e93f2b5fad:39921 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-16T14:32:51,203 INFO [RS:1;04e93f2b5fad:39921 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-16T14:32:51,203 INFO [RS:1;04e93f2b5fad:39921 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-16T14:32:51,204 INFO [RS:1;04e93f2b5fad:39921 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:39921 2024-11-16T14:32:51,204 DEBUG [RS:0;04e93f2b5fad:35241 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/0d13f5ec-97ce-91d4-7824-c5174956db56/oldWALs 2024-11-16T14:32:51,204 INFO [RS:0;04e93f2b5fad:35241 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 04e93f2b5fad%2C35241%2C1731767532873.meta:.meta(num 1731767535004) 2024-11-16T14:32:51,206 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43279 is added to blk_1073741834_1010 (size=1710) 2024-11-16T14:32:51,206 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36193 is added to blk_1073741834_1010 (size=1710) 2024-11-16T14:32:51,206 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33005 is added to blk_1073741834_1010 (size=1710) 2024-11-16T14:32:51,208 DEBUG [RS:0;04e93f2b5fad:35241 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/0d13f5ec-97ce-91d4-7824-c5174956db56/oldWALs 2024-11-16T14:32:51,208 INFO [RS:0;04e93f2b5fad:35241 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 04e93f2b5fad%2C35241%2C1731767532873:(num 1731767534512) 2024-11-16T14:32:51,208 DEBUG [RS:0;04e93f2b5fad:35241 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:32:51,208 INFO [RS:0;04e93f2b5fad:35241 {}] regionserver.LeaseManager(133): Closed leases 2024-11-16T14:32:51,208 INFO [RS:0;04e93f2b5fad:35241 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-16T14:32:51,209 INFO [RS:0;04e93f2b5fad:35241 {}] hbase.ChoreService(370): Chore service for: regionserver/04e93f2b5fad:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-16T14:32:51,209 INFO [RS:0;04e93f2b5fad:35241 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-16T14:32:51,209 INFO [regionserver/04e93f2b5fad:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-16T14:32:51,209 INFO [RS:0;04e93f2b5fad:35241 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:35241 2024-11-16T14:32:51,216 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34365-0x101440f72da0000, quorum=127.0.0.1:57212, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-16T14:32:51,216 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39921-0x101440f72da0002, quorum=127.0.0.1:57212, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/04e93f2b5fad,39921,1731767532970 2024-11-16T14:32:51,216 INFO [RS:1;04e93f2b5fad:39921 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-16T14:32:51,227 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35241-0x101440f72da0001, quorum=127.0.0.1:57212, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/04e93f2b5fad,35241,1731767532873 2024-11-16T14:32:51,227 INFO [RS:0;04e93f2b5fad:35241 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-16T14:32:51,237 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [04e93f2b5fad,39921,1731767532970] 2024-11-16T14:32:51,258 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/04e93f2b5fad,39921,1731767532970 already deleted, retry=false 2024-11-16T14:32:51,259 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 04e93f2b5fad,39921,1731767532970 expired; onlineServers=1 2024-11-16T14:32:51,259 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [04e93f2b5fad,35241,1731767532873] 2024-11-16T14:32:51,269 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/04e93f2b5fad,35241,1731767532873 already deleted, retry=false 2024-11-16T14:32:51,269 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 04e93f2b5fad,35241,1731767532873 expired; onlineServers=0 2024-11-16T14:32:51,269 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '04e93f2b5fad,34365,1731767532092' ***** 2024-11-16T14:32:51,270 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-16T14:32:51,270 INFO [M:0;04e93f2b5fad:34365 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-16T14:32:51,270 INFO [M:0;04e93f2b5fad:34365 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-16T14:32:51,270 DEBUG [M:0;04e93f2b5fad:34365 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-16T14:32:51,271 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-16T14:32:51,271 DEBUG [M:0;04e93f2b5fad:34365 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-16T14:32:51,271 DEBUG [master/04e93f2b5fad:0:becomeActiveMaster-HFileCleaner.large.0-1731767534162 {}] cleaner.HFileCleaner(306): Exit Thread[master/04e93f2b5fad:0:becomeActiveMaster-HFileCleaner.large.0-1731767534162,5,FailOnTimeoutGroup] 2024-11-16T14:32:51,271 DEBUG [master/04e93f2b5fad:0:becomeActiveMaster-HFileCleaner.small.0-1731767534165 {}] cleaner.HFileCleaner(306): Exit Thread[master/04e93f2b5fad:0:becomeActiveMaster-HFileCleaner.small.0-1731767534165,5,FailOnTimeoutGroup] 2024-11-16T14:32:51,271 INFO [M:0;04e93f2b5fad:34365 {}] hbase.ChoreService(370): Chore service for: master/04e93f2b5fad:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-16T14:32:51,272 INFO [M:0;04e93f2b5fad:34365 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-16T14:32:51,272 DEBUG [M:0;04e93f2b5fad:34365 {}] master.HMaster(1795): Stopping service threads 2024-11-16T14:32:51,272 INFO [M:0;04e93f2b5fad:34365 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-16T14:32:51,272 INFO [M:0;04e93f2b5fad:34365 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-16T14:32:51,273 INFO [M:0;04e93f2b5fad:34365 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-16T14:32:51,274 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-16T14:32:51,284 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34365-0x101440f72da0000, quorum=127.0.0.1:57212, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-16T14:32:51,284 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34365-0x101440f72da0000, quorum=127.0.0.1:57212, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T14:32:51,284 DEBUG [M:0;04e93f2b5fad:34365 {}] zookeeper.ZKUtil(347): master:34365-0x101440f72da0000, quorum=127.0.0.1:57212, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-16T14:32:51,284 WARN [M:0;04e93f2b5fad:34365 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-16T14:32:51,286 INFO [M:0;04e93f2b5fad:34365 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:43457/user/jenkins/test-data/0d13f5ec-97ce-91d4-7824-c5174956db56/.lastflushedseqids 2024-11-16T14:32:51,303 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43279 is added to blk_1073741941_1121 (size=138) 2024-11-16T14:32:51,303 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36193 is added to blk_1073741941_1121 (size=138) 2024-11-16T14:32:51,303 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33005 is added to blk_1073741941_1121 (size=138) 2024-11-16T14:32:51,303 INFO [M:0;04e93f2b5fad:34365 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-16T14:32:51,303 INFO [M:0;04e93f2b5fad:34365 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-16T14:32:51,304 DEBUG [M:0;04e93f2b5fad:34365 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-16T14:32:51,304 INFO [M:0;04e93f2b5fad:34365 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-16T14:32:51,304 DEBUG [M:0;04e93f2b5fad:34365 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-16T14:32:51,304 DEBUG [M:0;04e93f2b5fad:34365 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-16T14:32:51,304 DEBUG [M:0;04e93f2b5fad:34365 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-16T14:32:51,304 INFO [M:0;04e93f2b5fad:34365 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=68.36 KB heapSize=83.73 KB 2024-11-16T14:32:51,320 DEBUG [M:0;04e93f2b5fad:34365 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43457/user/jenkins/test-data/0d13f5ec-97ce-91d4-7824-c5174956db56/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/a2bdf2a3e38f4be68eabd2cb4954d2b3 is 82, key is hbase:meta,,1/info:regioninfo/1731767535090/Put/seqid=0 2024-11-16T14:32:51,325 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36193 is added to blk_1073741942_1122 (size=5672) 2024-11-16T14:32:51,325 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33005 is added to blk_1073741942_1122 (size=5672) 2024-11-16T14:32:51,325 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43279 is added to blk_1073741942_1122 (size=5672) 2024-11-16T14:32:51,325 INFO [M:0;04e93f2b5fad:34365 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:43457/user/jenkins/test-data/0d13f5ec-97ce-91d4-7824-c5174956db56/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/a2bdf2a3e38f4be68eabd2cb4954d2b3 2024-11-16T14:32:51,337 INFO [RS:1;04e93f2b5fad:39921 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-16T14:32:51,337 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39921-0x101440f72da0002, quorum=127.0.0.1:57212, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-16T14:32:51,337 INFO [RS:1;04e93f2b5fad:39921 {}] regionserver.HRegionServer(1031): Exiting; stopping=04e93f2b5fad,39921,1731767532970; zookeeper connection closed. 2024-11-16T14:32:51,337 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39921-0x101440f72da0002, quorum=127.0.0.1:57212, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-16T14:32:51,338 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@7eb03067 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@7eb03067 2024-11-16T14:32:51,345 DEBUG [M:0;04e93f2b5fad:34365 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43457/user/jenkins/test-data/0d13f5ec-97ce-91d4-7824-c5174956db56/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/eda401bd7fe34a1b9b08f18a49188ab9 is 1077, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1731767548184/Put/seqid=0 2024-11-16T14:32:51,348 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35241-0x101440f72da0001, quorum=127.0.0.1:57212, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-16T14:32:51,348 INFO [RS:0;04e93f2b5fad:35241 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-16T14:32:51,348 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35241-0x101440f72da0001, quorum=127.0.0.1:57212, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-16T14:32:51,348 INFO [RS:0;04e93f2b5fad:35241 {}] regionserver.HRegionServer(1031): Exiting; stopping=04e93f2b5fad,35241,1731767532873; zookeeper connection closed. 2024-11-16T14:32:51,348 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@27d7ec5f {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@27d7ec5f 2024-11-16T14:32:51,349 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 3 regionserver(s) complete 2024-11-16T14:32:51,350 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36193 is added to blk_1073741943_1123 (size=7756) 2024-11-16T14:32:51,350 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33005 is added to blk_1073741943_1123 (size=7756) 2024-11-16T14:32:51,351 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43279 is added to blk_1073741943_1123 (size=7756) 2024-11-16T14:32:51,351 INFO [M:0;04e93f2b5fad:34365 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=67.62 KB at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:43457/user/jenkins/test-data/0d13f5ec-97ce-91d4-7824-c5174956db56/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/eda401bd7fe34a1b9b08f18a49188ab9 2024-11-16T14:32:51,355 INFO [M:0;04e93f2b5fad:34365 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for eda401bd7fe34a1b9b08f18a49188ab9 2024-11-16T14:32:51,369 DEBUG [M:0;04e93f2b5fad:34365 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43457/user/jenkins/test-data/0d13f5ec-97ce-91d4-7824-c5174956db56/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/3762c20d45cc48e78ad21155c6189c78 is 69, key is 04e93f2b5fad,35241,1731767532873/rs:state/1731767534232/Put/seqid=0 2024-11-16T14:32:51,374 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33005 is added to blk_1073741944_1124 (size=5440) 2024-11-16T14:32:51,374 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36193 is added to blk_1073741944_1124 (size=5440) 2024-11-16T14:32:51,374 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43279 is added to blk_1073741944_1124 (size=5440) 2024-11-16T14:32:51,375 INFO [M:0;04e93f2b5fad:34365 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=249 B at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:43457/user/jenkins/test-data/0d13f5ec-97ce-91d4-7824-c5174956db56/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/3762c20d45cc48e78ad21155c6189c78 2024-11-16T14:32:51,379 INFO [M:0;04e93f2b5fad:34365 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 3762c20d45cc48e78ad21155c6189c78 2024-11-16T14:32:51,380 DEBUG [M:0;04e93f2b5fad:34365 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43457/user/jenkins/test-data/0d13f5ec-97ce-91d4-7824-c5174956db56/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/a2bdf2a3e38f4be68eabd2cb4954d2b3 as hdfs://localhost:43457/user/jenkins/test-data/0d13f5ec-97ce-91d4-7824-c5174956db56/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/a2bdf2a3e38f4be68eabd2cb4954d2b3 2024-11-16T14:32:51,385 INFO [M:0;04e93f2b5fad:34365 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43457/user/jenkins/test-data/0d13f5ec-97ce-91d4-7824-c5174956db56/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/a2bdf2a3e38f4be68eabd2cb4954d2b3, entries=8, sequenceid=168, filesize=5.5 K 2024-11-16T14:32:51,386 DEBUG [M:0;04e93f2b5fad:34365 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43457/user/jenkins/test-data/0d13f5ec-97ce-91d4-7824-c5174956db56/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/eda401bd7fe34a1b9b08f18a49188ab9 as hdfs://localhost:43457/user/jenkins/test-data/0d13f5ec-97ce-91d4-7824-c5174956db56/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/eda401bd7fe34a1b9b08f18a49188ab9 2024-11-16T14:32:51,390 INFO [M:0;04e93f2b5fad:34365 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for eda401bd7fe34a1b9b08f18a49188ab9 2024-11-16T14:32:51,390 INFO [M:0;04e93f2b5fad:34365 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43457/user/jenkins/test-data/0d13f5ec-97ce-91d4-7824-c5174956db56/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/eda401bd7fe34a1b9b08f18a49188ab9, entries=17, sequenceid=168, filesize=7.6 K 2024-11-16T14:32:51,391 DEBUG [M:0;04e93f2b5fad:34365 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43457/user/jenkins/test-data/0d13f5ec-97ce-91d4-7824-c5174956db56/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/3762c20d45cc48e78ad21155c6189c78 as hdfs://localhost:43457/user/jenkins/test-data/0d13f5ec-97ce-91d4-7824-c5174956db56/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/3762c20d45cc48e78ad21155c6189c78 2024-11-16T14:32:51,394 INFO [M:0;04e93f2b5fad:34365 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 3762c20d45cc48e78ad21155c6189c78 2024-11-16T14:32:51,394 INFO [M:0;04e93f2b5fad:34365 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43457/user/jenkins/test-data/0d13f5ec-97ce-91d4-7824-c5174956db56/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/3762c20d45cc48e78ad21155c6189c78, entries=3, sequenceid=168, filesize=5.3 K 2024-11-16T14:32:51,395 INFO [M:0;04e93f2b5fad:34365 {}] regionserver.HRegion(3140): Finished flush of dataSize ~68.36 KB/69996, heapSize ~83.44 KB/85440, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 91ms, sequenceid=168, compaction requested=false 2024-11-16T14:32:51,397 INFO [M:0;04e93f2b5fad:34365 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-16T14:32:51,397 DEBUG [M:0;04e93f2b5fad:34365 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731767571303Disabling compacts and flushes for region at 1731767571303Disabling writes for close at 1731767571304 (+1 ms)Obtaining lock to block concurrent updates at 1731767571304Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731767571304Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=69996, getHeapSize=85680, getOffHeapSize=0, getCellsCount=195 at 1731767571304Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731767571305 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731767571305Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731767571319 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731767571319Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731767571329 (+10 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731767571344 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731767571344Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731767571355 (+11 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731767571368 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731767571368Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@24b5ec33: reopening flushed file at 1731767571379 (+11 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3ee256ba: reopening flushed file at 1731767571385 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@57e58800: reopening flushed file at 1731767571390 (+5 ms)Finished flush of dataSize ~68.36 KB/69996, heapSize ~83.44 KB/85440, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 91ms, sequenceid=168, compaction requested=false at 1731767571395 (+5 ms)Writing region close event to WAL at 1731767571396 (+1 ms)Closed at 1731767571396 2024-11-16T14:32:51,399 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33005 is added to blk_1073741830_1006 (size=56614) 2024-11-16T14:32:51,399 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36193 is added to blk_1073741830_1006 (size=56614) 2024-11-16T14:32:51,399 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43279 is added to blk_1073741830_1006 (size=56614) 2024-11-16T14:32:51,399 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-16T14:32:51,400 INFO [M:0;04e93f2b5fad:34365 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-16T14:32:51,400 INFO [M:0;04e93f2b5fad:34365 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:34365 2024-11-16T14:32:51,400 INFO [M:0;04e93f2b5fad:34365 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-16T14:32:51,510 INFO [M:0;04e93f2b5fad:34365 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-16T14:32:51,510 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34365-0x101440f72da0000, quorum=127.0.0.1:57212, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-16T14:32:51,510 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34365-0x101440f72da0000, quorum=127.0.0.1:57212, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-16T14:32:51,516 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731767551199/wal.1731767551590 with renewLeaseKey: DEFAULT_16655 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731767551199/wal.1731767551590 (inode 16655) Holder DFSClient_NONMAPREDUCE_1995475656_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731767551199/wal.1731767551590 (inode 16655) Holder DFSClient_NONMAPREDUCE_1995475656_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-11-16T14:32:51,516 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731767543287/wal.1731767543437 with renewLeaseKey: DEFAULT_16586 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:32:51,517 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731767566288/wal.1731767566466 with renewLeaseKey: DEFAULT_16767 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:32:51,520 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731767551733/wal.1731767561082 with renewLeaseKey: DEFAULT_16678 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731767551733/wal.1731767561082 (inode 16678) Holder DFSClient_NONMAPREDUCE_1995475656_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731767551733/wal.1731767561082 (inode 16678) Holder DFSClient_NONMAPREDUCE_1995475656_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-11-16T14:32:51,521 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731767536215/wal.1731767536284 with renewLeaseKey: DEFAULT_16506 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:32:51,523 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testdatalosswheninputerror-manual,16010,1731767561264/wal.1731767561739 with renewLeaseKey: DEFAULT_16704 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testdatalosswheninputerror-manual,16010,1731767561264/wal.1731767561739 (inode 16704) Holder DFSClient_NONMAPREDUCE_1995475656_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testdatalosswheninputerror-manual,16010,1731767561264/wal.1731767561739 (inode 16704) Holder DFSClient_NONMAPREDUCE_1995475656_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-11-16T14:32:51,523 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731767561886/wal.1731767561947 with renewLeaseKey: DEFAULT_16726 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:32:51,525 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testnameconflictwhensplit1-manual,16010,1731767535895/wal.1731767536076 with renewLeaseKey: DEFAULT_16485 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testnameconflictwhensplit1-manual,16010,1731767535895/wal.1731767536076 (inode 16485) Holder DFSClient_NONMAPREDUCE_1995475656_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testnameconflictwhensplit1-manual,16010,1731767535895/wal.1731767536076 (inode 16485) Holder DFSClient_NONMAPREDUCE_1995475656_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-11-16T14:32:51,528 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testnameconflictwhensplit0-manual,16010,1731767535448/wal.1731767535668 with renewLeaseKey: DEFAULT_16462 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testnameconflictwhensplit0-manual,16010,1731767535448/wal.1731767535668 (inode 16462) Holder DFSClient_NONMAPREDUCE_1995475656_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testnameconflictwhensplit0-manual,16010,1731767535448/wal.1731767535668 (inode 16462) Holder DFSClient_NONMAPREDUCE_1995475656_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-11-16T14:32:51,578 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@35f1150e{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-16T14:32:51,582 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@13a77e13{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-16T14:32:51,582 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-16T14:32:51,583 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@f9972d0{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-16T14:32:51,583 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@2a6d5e13{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/650842a5-0ab8-4683-35e8-38af23b31d67/hadoop.log.dir/,STOPPED} 2024-11-16T14:32:51,586 WARN [BP-865919460-172.17.0.2-1731767527819 heartbeating to localhost/127.0.0.1:43457 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-16T14:32:51,586 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-16T14:32:51,586 WARN [BP-865919460-172.17.0.2-1731767527819 heartbeating to localhost/127.0.0.1:43457 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-865919460-172.17.0.2-1731767527819 (Datanode Uuid a4e7ac56-98d1-4613-9843-f3379d05420b) service to localhost/127.0.0.1:43457 2024-11-16T14:32:51,586 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-16T14:32:51,588 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/650842a5-0ab8-4683-35e8-38af23b31d67/cluster_174e5b33-598d-20b3-a770-4fb31341c285/data/data5/current/BP-865919460-172.17.0.2-1731767527819 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-16T14:32:51,588 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/650842a5-0ab8-4683-35e8-38af23b31d67/cluster_174e5b33-598d-20b3-a770-4fb31341c285/data/data6/current/BP-865919460-172.17.0.2-1731767527819 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-16T14:32:51,588 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-16T14:32:51,590 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@7bd427b8{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-16T14:32:51,590 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@6915083f{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-16T14:32:51,590 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-16T14:32:51,590 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@5cc2d6b{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-16T14:32:51,590 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@46b092e1{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/650842a5-0ab8-4683-35e8-38af23b31d67/hadoop.log.dir/,STOPPED} 2024-11-16T14:32:51,591 WARN [BP-865919460-172.17.0.2-1731767527819 heartbeating to localhost/127.0.0.1:43457 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-16T14:32:51,591 WARN [BP-865919460-172.17.0.2-1731767527819 heartbeating to localhost/127.0.0.1:43457 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-865919460-172.17.0.2-1731767527819 (Datanode Uuid 870c4796-08a6-4764-94fb-ce871302a15a) service to localhost/127.0.0.1:43457 2024-11-16T14:32:51,591 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-16T14:32:51,591 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-16T14:32:51,592 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/650842a5-0ab8-4683-35e8-38af23b31d67/cluster_174e5b33-598d-20b3-a770-4fb31341c285/data/data3/current/BP-865919460-172.17.0.2-1731767527819 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-16T14:32:51,592 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/650842a5-0ab8-4683-35e8-38af23b31d67/cluster_174e5b33-598d-20b3-a770-4fb31341c285/data/data4/current/BP-865919460-172.17.0.2-1731767527819 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-16T14:32:51,592 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-16T14:32:51,594 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@330740de{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-16T14:32:51,594 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@7b24cab9{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-16T14:32:51,595 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-16T14:32:51,595 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4a359997{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-16T14:32:51,595 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@cf5a85e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/650842a5-0ab8-4683-35e8-38af23b31d67/hadoop.log.dir/,STOPPED} 2024-11-16T14:32:51,596 WARN [BP-865919460-172.17.0.2-1731767527819 heartbeating to localhost/127.0.0.1:43457 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-16T14:32:51,596 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-16T14:32:51,596 WARN [BP-865919460-172.17.0.2-1731767527819 heartbeating to localhost/127.0.0.1:43457 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-865919460-172.17.0.2-1731767527819 (Datanode Uuid da003da4-b43a-4f8f-a16e-388e885fb77e) service to localhost/127.0.0.1:43457 2024-11-16T14:32:51,596 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-16T14:32:51,597 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/650842a5-0ab8-4683-35e8-38af23b31d67/cluster_174e5b33-598d-20b3-a770-4fb31341c285/data/data1/current/BP-865919460-172.17.0.2-1731767527819 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-16T14:32:51,597 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/650842a5-0ab8-4683-35e8-38af23b31d67/cluster_174e5b33-598d-20b3-a770-4fb31341c285/data/data2/current/BP-865919460-172.17.0.2-1731767527819 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-16T14:32:51,597 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-16T14:32:51,603 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@3717288f{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-16T14:32:51,604 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@4bd70930{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-16T14:32:51,604 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-16T14:32:51,604 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6dc9d5c1{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-16T14:32:51,605 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4f37ffca{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/650842a5-0ab8-4683-35e8-38af23b31d67/hadoop.log.dir/,STOPPED} 2024-11-16T14:32:51,619 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-16T14:32:51,673 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down