2024-11-12 14:43:14,689 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@20b2475a 2024-11-12 14:43:14,705 main DEBUG Took 0.014178 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-12 14:43:14,706 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-12 14:43:14,706 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-12 14:43:14,708 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-12 14:43:14,710 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-12 14:43:14,721 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-12 14:43:14,737 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 14:43:14,739 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-12 14:43:14,740 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 14:43:14,741 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-12 14:43:14,742 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 14:43:14,742 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-12 14:43:14,744 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 14:43:14,745 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-12 14:43:14,745 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 14:43:14,746 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-12 14:43:14,747 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 14:43:14,747 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-12 14:43:14,748 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 14:43:14,749 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-12 14:43:14,750 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 14:43:14,750 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-12 14:43:14,751 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 14:43:14,751 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-12 14:43:14,752 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 14:43:14,753 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-12 14:43:14,753 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 14:43:14,754 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-12 14:43:14,754 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 14:43:14,755 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-12 14:43:14,756 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 14:43:14,756 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-12 14:43:14,758 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 14:43:14,760 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-12 14:43:14,762 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-12 14:43:14,763 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-12 14:43:14,764 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-12 14:43:14,765 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-12 14:43:14,776 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-12 14:43:14,779 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-12 14:43:14,781 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-12 14:43:14,781 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-12 14:43:14,782 main DEBUG createAppenders(={Console}) 2024-11-12 14:43:14,783 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@20b2475a initialized 2024-11-12 14:43:14,783 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@20b2475a 2024-11-12 14:43:14,783 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@20b2475a OK. 2024-11-12 14:43:14,783 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-12 14:43:14,784 main DEBUG OutputStream closed 2024-11-12 14:43:14,784 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-12 14:43:14,784 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-12 14:43:14,784 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@4310d43 OK 2024-11-12 14:43:14,852 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-12 14:43:14,854 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-12 14:43:14,855 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-12 14:43:14,856 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-12 14:43:14,856 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-12 14:43:14,856 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-12 14:43:14,857 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-12 14:43:14,857 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-12 14:43:14,857 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-12 14:43:14,857 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-12 14:43:14,858 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-12 14:43:14,858 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-12 14:43:14,858 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-12 14:43:14,859 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-12 14:43:14,859 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-12 14:43:14,859 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-12 14:43:14,859 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-12 14:43:14,860 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-12 14:43:14,862 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-12 14:43:14,862 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@12f9af83) with optional ClassLoader: null 2024-11-12 14:43:14,863 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-12 14:43:14,863 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@12f9af83] started OK. 2024-11-12T14:43:15,120 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5a5e6c5c-3b82-c3b3-1527-3525a941b154 2024-11-12 14:43:15,122 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-12 14:43:15,123 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-12T14:43:15,131 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay timeout: 13 mins 2024-11-12T14:43:15,137 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.regionserver.wal.TestSecureAsyncWALReplay timeout: 13 mins 2024-11-12T14:43:15,165 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-12T14:43:15,210 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-12T14:43:15,210 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-12T14:43:15,223 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=3, rsPorts=, rsClass=null, numDataNodes=3, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-12T14:43:15,242 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5a5e6c5c-3b82-c3b3-1527-3525a941b154/cluster_22b4e46d-f388-f2af-b32c-f9fd886af812, deleteOnExit=true 2024-11-12T14:43:15,243 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-12T14:43:15,244 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5a5e6c5c-3b82-c3b3-1527-3525a941b154/test.cache.data in system properties and HBase conf 2024-11-12T14:43:15,244 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5a5e6c5c-3b82-c3b3-1527-3525a941b154/hadoop.tmp.dir in system properties and HBase conf 2024-11-12T14:43:15,245 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5a5e6c5c-3b82-c3b3-1527-3525a941b154/hadoop.log.dir in system properties and HBase conf 2024-11-12T14:43:15,246 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5a5e6c5c-3b82-c3b3-1527-3525a941b154/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-12T14:43:15,246 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5a5e6c5c-3b82-c3b3-1527-3525a941b154/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-12T14:43:15,246 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-12T14:43:15,320 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-12T14:43:15,408 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-12T14:43:15,413 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5a5e6c5c-3b82-c3b3-1527-3525a941b154/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-12T14:43:15,414 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5a5e6c5c-3b82-c3b3-1527-3525a941b154/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-12T14:43:15,415 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5a5e6c5c-3b82-c3b3-1527-3525a941b154/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-12T14:43:15,416 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5a5e6c5c-3b82-c3b3-1527-3525a941b154/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-12T14:43:15,416 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5a5e6c5c-3b82-c3b3-1527-3525a941b154/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-12T14:43:15,417 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5a5e6c5c-3b82-c3b3-1527-3525a941b154/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-12T14:43:15,418 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5a5e6c5c-3b82-c3b3-1527-3525a941b154/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-12T14:43:15,418 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5a5e6c5c-3b82-c3b3-1527-3525a941b154/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-12T14:43:15,419 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5a5e6c5c-3b82-c3b3-1527-3525a941b154/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-12T14:43:15,420 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5a5e6c5c-3b82-c3b3-1527-3525a941b154/nfs.dump.dir in system properties and HBase conf 2024-11-12T14:43:15,421 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5a5e6c5c-3b82-c3b3-1527-3525a941b154/java.io.tmpdir in system properties and HBase conf 2024-11-12T14:43:15,421 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5a5e6c5c-3b82-c3b3-1527-3525a941b154/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-12T14:43:15,422 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5a5e6c5c-3b82-c3b3-1527-3525a941b154/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-12T14:43:15,423 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5a5e6c5c-3b82-c3b3-1527-3525a941b154/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-12T14:43:16,287 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-12T14:43:16,391 INFO [Time-limited test {}] log.Log(170): Logging initialized @2428ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-12T14:43:16,507 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-12T14:43:16,608 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-12T14:43:16,648 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-12T14:43:16,648 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-12T14:43:16,650 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-12T14:43:16,673 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-12T14:43:16,678 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6bdd7391{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5a5e6c5c-3b82-c3b3-1527-3525a941b154/hadoop.log.dir/,AVAILABLE} 2024-11-12T14:43:16,681 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4e578b56{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-12T14:43:16,916 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@6878e65d{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5a5e6c5c-3b82-c3b3-1527-3525a941b154/java.io.tmpdir/jetty-localhost-38407-hadoop-hdfs-3_4_1-tests_jar-_-any-9322020958680335056/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-12T14:43:16,924 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@2f30922e{HTTP/1.1, (http/1.1)}{localhost:38407} 2024-11-12T14:43:16,924 INFO [Time-limited test {}] server.Server(415): Started @2961ms 2024-11-12T14:43:17,254 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-12T14:43:17,262 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-12T14:43:17,263 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-12T14:43:17,263 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-12T14:43:17,264 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-12T14:43:17,264 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3f494b95{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5a5e6c5c-3b82-c3b3-1527-3525a941b154/hadoop.log.dir/,AVAILABLE} 2024-11-12T14:43:17,265 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@63c29ae0{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-12T14:43:17,371 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@388bacc3{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5a5e6c5c-3b82-c3b3-1527-3525a941b154/java.io.tmpdir/jetty-localhost-41807-hadoop-hdfs-3_4_1-tests_jar-_-any-3632024595356107867/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-12T14:43:17,372 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@6f48fbbe{HTTP/1.1, (http/1.1)}{localhost:41807} 2024-11-12T14:43:17,372 INFO [Time-limited test {}] server.Server(415): Started @3409ms 2024-11-12T14:43:17,423 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-12T14:43:17,539 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-12T14:43:17,546 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-12T14:43:17,551 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-12T14:43:17,552 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-12T14:43:17,552 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-12T14:43:17,553 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3fe9d746{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5a5e6c5c-3b82-c3b3-1527-3525a941b154/hadoop.log.dir/,AVAILABLE} 2024-11-12T14:43:17,554 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@540114bd{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-12T14:43:17,681 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@565a2c18{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5a5e6c5c-3b82-c3b3-1527-3525a941b154/java.io.tmpdir/jetty-localhost-44011-hadoop-hdfs-3_4_1-tests_jar-_-any-16266614058275247317/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-12T14:43:17,682 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@311fb016{HTTP/1.1, (http/1.1)}{localhost:44011} 2024-11-12T14:43:17,683 INFO [Time-limited test {}] server.Server(415): Started @3720ms 2024-11-12T14:43:17,685 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-12T14:43:17,735 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-12T14:43:17,743 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-12T14:43:17,744 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-12T14:43:17,744 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-12T14:43:17,744 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-12T14:43:17,745 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4d7caafc{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5a5e6c5c-3b82-c3b3-1527-3525a941b154/hadoop.log.dir/,AVAILABLE} 2024-11-12T14:43:17,746 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@606c1a1{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-12T14:43:17,834 WARN [Thread-108 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5a5e6c5c-3b82-c3b3-1527-3525a941b154/cluster_22b4e46d-f388-f2af-b32c-f9fd886af812/data/data2/current/BP-1980882744-172.17.0.2-1731422596004/current, will proceed with Du for space computation calculation, 2024-11-12T14:43:17,834 WARN [Thread-106 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5a5e6c5c-3b82-c3b3-1527-3525a941b154/cluster_22b4e46d-f388-f2af-b32c-f9fd886af812/data/data1/current/BP-1980882744-172.17.0.2-1731422596004/current, will proceed with Du for space computation calculation, 2024-11-12T14:43:17,834 WARN [Thread-107 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5a5e6c5c-3b82-c3b3-1527-3525a941b154/cluster_22b4e46d-f388-f2af-b32c-f9fd886af812/data/data4/current/BP-1980882744-172.17.0.2-1731422596004/current, will proceed with Du for space computation calculation, 2024-11-12T14:43:17,834 WARN [Thread-105 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5a5e6c5c-3b82-c3b3-1527-3525a941b154/cluster_22b4e46d-f388-f2af-b32c-f9fd886af812/data/data3/current/BP-1980882744-172.17.0.2-1731422596004/current, will proceed with Du for space computation calculation, 2024-11-12T14:43:17,871 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@b7f81b{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5a5e6c5c-3b82-c3b3-1527-3525a941b154/java.io.tmpdir/jetty-localhost-46263-hadoop-hdfs-3_4_1-tests_jar-_-any-11873903575704859446/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-12T14:43:17,872 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@5d3aa315{HTTP/1.1, (http/1.1)}{localhost:46263} 2024-11-12T14:43:17,872 INFO [Time-limited test {}] server.Server(415): Started @3909ms 2024-11-12T14:43:17,875 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-12T14:43:17,878 WARN [Thread-82 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-12T14:43:17,879 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-12T14:43:17,948 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x185c1ce207b84a12 with lease ID 0xfc30ab20f9979540: Processing first storage report for DS-31e7ba0a-e37c-4fe6-8d2a-a4846cefa408 from datanode DatanodeRegistration(127.0.0.1:36175, datanodeUuid=46e43ae4-d140-41eb-8f46-d79b5d55189d, infoPort=36027, infoSecurePort=0, ipcPort=35357, storageInfo=lv=-57;cid=testClusterID;nsid=968263674;c=1731422596004) 2024-11-12T14:43:17,950 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x185c1ce207b84a12 with lease ID 0xfc30ab20f9979540: from storage DS-31e7ba0a-e37c-4fe6-8d2a-a4846cefa408 node DatanodeRegistration(127.0.0.1:36175, datanodeUuid=46e43ae4-d140-41eb-8f46-d79b5d55189d, infoPort=36027, infoSecurePort=0, ipcPort=35357, storageInfo=lv=-57;cid=testClusterID;nsid=968263674;c=1731422596004), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-12T14:43:17,950 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x382d74be652ff021 with lease ID 0xfc30ab20f997953f: Processing first storage report for DS-80b047b6-34d2-41c0-99c3-5cd61ca40075 from datanode DatanodeRegistration(127.0.0.1:45975, datanodeUuid=9aab8b87-083c-4077-8d4b-86455c74734f, infoPort=43643, infoSecurePort=0, ipcPort=41811, storageInfo=lv=-57;cid=testClusterID;nsid=968263674;c=1731422596004) 2024-11-12T14:43:17,951 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x382d74be652ff021 with lease ID 0xfc30ab20f997953f: from storage DS-80b047b6-34d2-41c0-99c3-5cd61ca40075 node DatanodeRegistration(127.0.0.1:45975, datanodeUuid=9aab8b87-083c-4077-8d4b-86455c74734f, infoPort=43643, infoSecurePort=0, ipcPort=41811, storageInfo=lv=-57;cid=testClusterID;nsid=968263674;c=1731422596004), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-12T14:43:17,951 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x185c1ce207b84a12 with lease ID 0xfc30ab20f9979540: Processing first storage report for DS-4a58ebac-4435-4aed-a0b1-ef4d4d024be0 from datanode DatanodeRegistration(127.0.0.1:36175, datanodeUuid=46e43ae4-d140-41eb-8f46-d79b5d55189d, infoPort=36027, infoSecurePort=0, ipcPort=35357, storageInfo=lv=-57;cid=testClusterID;nsid=968263674;c=1731422596004) 2024-11-12T14:43:17,951 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x185c1ce207b84a12 with lease ID 0xfc30ab20f9979540: from storage DS-4a58ebac-4435-4aed-a0b1-ef4d4d024be0 node DatanodeRegistration(127.0.0.1:36175, datanodeUuid=46e43ae4-d140-41eb-8f46-d79b5d55189d, infoPort=36027, infoSecurePort=0, ipcPort=35357, storageInfo=lv=-57;cid=testClusterID;nsid=968263674;c=1731422596004), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-12T14:43:17,952 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x382d74be652ff021 with lease ID 0xfc30ab20f997953f: Processing first storage report for DS-f64c27f7-ca67-41ed-9af4-54a2033009f1 from datanode DatanodeRegistration(127.0.0.1:45975, datanodeUuid=9aab8b87-083c-4077-8d4b-86455c74734f, infoPort=43643, infoSecurePort=0, ipcPort=41811, storageInfo=lv=-57;cid=testClusterID;nsid=968263674;c=1731422596004) 2024-11-12T14:43:17,952 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x382d74be652ff021 with lease ID 0xfc30ab20f997953f: from storage DS-f64c27f7-ca67-41ed-9af4-54a2033009f1 node DatanodeRegistration(127.0.0.1:45975, datanodeUuid=9aab8b87-083c-4077-8d4b-86455c74734f, infoPort=43643, infoSecurePort=0, ipcPort=41811, storageInfo=lv=-57;cid=testClusterID;nsid=968263674;c=1731422596004), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-12T14:43:17,984 WARN [Thread-139 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5a5e6c5c-3b82-c3b3-1527-3525a941b154/cluster_22b4e46d-f388-f2af-b32c-f9fd886af812/data/data5/current/BP-1980882744-172.17.0.2-1731422596004/current, will proceed with Du for space computation calculation, 2024-11-12T14:43:17,984 WARN [Thread-140 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5a5e6c5c-3b82-c3b3-1527-3525a941b154/cluster_22b4e46d-f388-f2af-b32c-f9fd886af812/data/data6/current/BP-1980882744-172.17.0.2-1731422596004/current, will proceed with Du for space computation calculation, 2024-11-12T14:43:18,016 WARN [Thread-129 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-12T14:43:18,024 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x4578ae44b9795f54 with lease ID 0xfc30ab20f9979541: Processing first storage report for DS-7317e4d9-cef6-4b85-a947-106c3c917da9 from datanode DatanodeRegistration(127.0.0.1:37395, datanodeUuid=b4a8cc10-901f-4240-88db-9ed40bfbd283, infoPort=40227, infoSecurePort=0, ipcPort=44907, storageInfo=lv=-57;cid=testClusterID;nsid=968263674;c=1731422596004) 2024-11-12T14:43:18,024 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x4578ae44b9795f54 with lease ID 0xfc30ab20f9979541: from storage DS-7317e4d9-cef6-4b85-a947-106c3c917da9 node DatanodeRegistration(127.0.0.1:37395, datanodeUuid=b4a8cc10-901f-4240-88db-9ed40bfbd283, infoPort=40227, infoSecurePort=0, ipcPort=44907, storageInfo=lv=-57;cid=testClusterID;nsid=968263674;c=1731422596004), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-12T14:43:18,024 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x4578ae44b9795f54 with lease ID 0xfc30ab20f9979541: Processing first storage report for DS-6cfa5cdc-cbd0-4234-9206-22f06a6fb62e from datanode DatanodeRegistration(127.0.0.1:37395, datanodeUuid=b4a8cc10-901f-4240-88db-9ed40bfbd283, infoPort=40227, infoSecurePort=0, ipcPort=44907, storageInfo=lv=-57;cid=testClusterID;nsid=968263674;c=1731422596004) 2024-11-12T14:43:18,024 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x4578ae44b9795f54 with lease ID 0xfc30ab20f9979541: from storage DS-6cfa5cdc-cbd0-4234-9206-22f06a6fb62e node DatanodeRegistration(127.0.0.1:37395, datanodeUuid=b4a8cc10-901f-4240-88db-9ed40bfbd283, infoPort=40227, infoSecurePort=0, ipcPort=44907, storageInfo=lv=-57;cid=testClusterID;nsid=968263674;c=1731422596004), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-12T14:43:18,212 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5a5e6c5c-3b82-c3b3-1527-3525a941b154 2024-11-12T14:43:18,299 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5a5e6c5c-3b82-c3b3-1527-3525a941b154/cluster_22b4e46d-f388-f2af-b32c-f9fd886af812/zookeeper_0, clientPort=55341, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5a5e6c5c-3b82-c3b3-1527-3525a941b154/cluster_22b4e46d-f388-f2af-b32c-f9fd886af812/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5a5e6c5c-3b82-c3b3-1527-3525a941b154/cluster_22b4e46d-f388-f2af-b32c-f9fd886af812/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-12T14:43:18,309 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=55341 2024-11-12T14:43:18,327 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-12T14:43:18,330 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-12T14:43:18,516 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37395 is added to blk_1073741825_1001 (size=7) 2024-11-12T14:43:18,517 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45975 is added to blk_1073741825_1001 (size=7) 2024-11-12T14:43:18,517 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36175 is added to blk_1073741825_1001 (size=7) 2024-11-12T14:43:18,928 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:40559/user/jenkins/test-data/41a388f8-1764-825e-e285-25964d37c9f3 with version=8 2024-11-12T14:43:18,928 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:40559/user/jenkins/test-data/41a388f8-1764-825e-e285-25964d37c9f3/hbase-staging 2024-11-12T14:43:19,167 INFO [Time-limited test {}] client.ConnectionUtils(128): master/3945e1ca6f47:0 server-side Connection retries=45 2024-11-12T14:43:19,175 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-12T14:43:19,176 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-12T14:43:19,180 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-12T14:43:19,180 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-12T14:43:19,180 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-12T14:43:19,312 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-12T14:43:19,369 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-12T14:43:19,378 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-12T14:43:19,381 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-12T14:43:19,407 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 10288 (auto-detected) 2024-11-12T14:43:19,408 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-12T14:43:19,429 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:34233 2024-11-12T14:43:19,451 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:34233 connecting to ZooKeeper ensemble=127.0.0.1:55341 2024-11-12T14:43:19,478 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:342330x0, quorum=127.0.0.1:55341, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-12T14:43:19,481 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:34233-0x100346390eb0000 connected 2024-11-12T14:43:19,508 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-12T14:43:19,511 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-12T14:43:19,521 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:34233-0x100346390eb0000, quorum=127.0.0.1:55341, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-12T14:43:19,524 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:40559/user/jenkins/test-data/41a388f8-1764-825e-e285-25964d37c9f3, hbase.cluster.distributed=false 2024-11-12T14:43:19,547 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:34233-0x100346390eb0000, quorum=127.0.0.1:55341, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-12T14:43:19,551 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=34233 2024-11-12T14:43:19,552 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=34233 2024-11-12T14:43:19,552 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=34233 2024-11-12T14:43:19,553 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=34233 2024-11-12T14:43:19,553 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=34233 2024-11-12T14:43:19,643 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/3945e1ca6f47:0 server-side Connection retries=45 2024-11-12T14:43:19,644 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-12T14:43:19,645 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-12T14:43:19,645 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-12T14:43:19,645 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-12T14:43:19,645 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-12T14:43:19,647 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-12T14:43:19,650 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-12T14:43:19,651 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:41431 2024-11-12T14:43:19,654 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:41431 connecting to ZooKeeper ensemble=127.0.0.1:55341 2024-11-12T14:43:19,655 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-12T14:43:19,660 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-12T14:43:19,665 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:414310x0, quorum=127.0.0.1:55341, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-12T14:43:19,666 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:41431-0x100346390eb0001 connected 2024-11-12T14:43:19,666 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:41431-0x100346390eb0001, quorum=127.0.0.1:55341, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-12T14:43:19,670 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-12T14:43:19,676 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-12T14:43:19,679 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:41431-0x100346390eb0001, quorum=127.0.0.1:55341, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-12T14:43:19,683 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:41431-0x100346390eb0001, quorum=127.0.0.1:55341, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-12T14:43:19,684 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=41431 2024-11-12T14:43:19,684 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=41431 2024-11-12T14:43:19,684 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=41431 2024-11-12T14:43:19,685 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=41431 2024-11-12T14:43:19,685 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=41431 2024-11-12T14:43:19,700 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/3945e1ca6f47:0 server-side Connection retries=45 2024-11-12T14:43:19,700 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-12T14:43:19,700 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-12T14:43:19,701 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-12T14:43:19,701 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-12T14:43:19,701 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-12T14:43:19,701 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-12T14:43:19,702 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-12T14:43:19,702 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:33387 2024-11-12T14:43:19,704 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:33387 connecting to ZooKeeper ensemble=127.0.0.1:55341 2024-11-12T14:43:19,705 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-12T14:43:19,707 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-12T14:43:19,712 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:333870x0, quorum=127.0.0.1:55341, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-12T14:43:19,713 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:333870x0, quorum=127.0.0.1:55341, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-12T14:43:19,713 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:33387-0x100346390eb0002 connected 2024-11-12T14:43:19,713 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-12T14:43:19,714 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-12T14:43:19,715 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:33387-0x100346390eb0002, quorum=127.0.0.1:55341, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-12T14:43:19,716 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:33387-0x100346390eb0002, quorum=127.0.0.1:55341, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-12T14:43:19,717 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=33387 2024-11-12T14:43:19,717 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=33387 2024-11-12T14:43:19,721 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=33387 2024-11-12T14:43:19,722 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=33387 2024-11-12T14:43:19,722 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=33387 2024-11-12T14:43:19,738 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/3945e1ca6f47:0 server-side Connection retries=45 2024-11-12T14:43:19,738 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-12T14:43:19,739 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-12T14:43:19,739 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-12T14:43:19,739 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-12T14:43:19,739 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-12T14:43:19,739 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-12T14:43:19,740 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-12T14:43:19,741 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:38727 2024-11-12T14:43:19,744 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:38727 connecting to ZooKeeper ensemble=127.0.0.1:55341 2024-11-12T14:43:19,745 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-12T14:43:19,748 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-12T14:43:19,753 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:387270x0, quorum=127.0.0.1:55341, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-12T14:43:19,754 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:387270x0, quorum=127.0.0.1:55341, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-12T14:43:19,755 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-12T14:43:19,755 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:38727-0x100346390eb0003 connected 2024-11-12T14:43:19,757 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-12T14:43:19,758 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38727-0x100346390eb0003, quorum=127.0.0.1:55341, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-12T14:43:19,760 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38727-0x100346390eb0003, quorum=127.0.0.1:55341, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-12T14:43:19,760 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=38727 2024-11-12T14:43:19,761 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=38727 2024-11-12T14:43:19,761 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=38727 2024-11-12T14:43:19,762 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=38727 2024-11-12T14:43:19,762 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=38727 2024-11-12T14:43:19,775 DEBUG [M:0;3945e1ca6f47:34233 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;3945e1ca6f47:34233 2024-11-12T14:43:19,776 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/3945e1ca6f47,34233,1731422599017 2024-11-12T14:43:19,783 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34233-0x100346390eb0000, quorum=127.0.0.1:55341, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-12T14:43:19,783 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41431-0x100346390eb0001, quorum=127.0.0.1:55341, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-12T14:43:19,783 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38727-0x100346390eb0003, quorum=127.0.0.1:55341, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-12T14:43:19,783 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33387-0x100346390eb0002, quorum=127.0.0.1:55341, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-12T14:43:19,785 DEBUG [master/3945e1ca6f47:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:34233-0x100346390eb0000, quorum=127.0.0.1:55341, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/3945e1ca6f47,34233,1731422599017 2024-11-12T14:43:19,809 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41431-0x100346390eb0001, quorum=127.0.0.1:55341, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-12T14:43:19,809 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38727-0x100346390eb0003, quorum=127.0.0.1:55341, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-12T14:43:19,810 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34233-0x100346390eb0000, quorum=127.0.0.1:55341, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T14:43:19,810 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38727-0x100346390eb0003, quorum=127.0.0.1:55341, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T14:43:19,809 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33387-0x100346390eb0002, quorum=127.0.0.1:55341, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-12T14:43:19,810 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41431-0x100346390eb0001, quorum=127.0.0.1:55341, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T14:43:19,810 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33387-0x100346390eb0002, quorum=127.0.0.1:55341, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T14:43:19,810 DEBUG [master/3945e1ca6f47:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:34233-0x100346390eb0000, quorum=127.0.0.1:55341, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-12T14:43:19,812 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/3945e1ca6f47,34233,1731422599017 from backup master directory 2024-11-12T14:43:19,814 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34233-0x100346390eb0000, quorum=127.0.0.1:55341, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/3945e1ca6f47,34233,1731422599017 2024-11-12T14:43:19,814 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41431-0x100346390eb0001, quorum=127.0.0.1:55341, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-12T14:43:19,814 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33387-0x100346390eb0002, quorum=127.0.0.1:55341, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-12T14:43:19,814 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34233-0x100346390eb0000, quorum=127.0.0.1:55341, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-12T14:43:19,814 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38727-0x100346390eb0003, quorum=127.0.0.1:55341, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-12T14:43:19,815 WARN [master/3945e1ca6f47:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-12T14:43:19,815 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=3945e1ca6f47,34233,1731422599017 2024-11-12T14:43:19,817 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-12T14:43:19,818 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-12T14:43:19,873 DEBUG [master/3945e1ca6f47:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:40559/user/jenkins/test-data/41a388f8-1764-825e-e285-25964d37c9f3/hbase.id] with ID: 70468ada-4082-4450-9033-a07d7e00c036 2024-11-12T14:43:19,873 DEBUG [master/3945e1ca6f47:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:40559/user/jenkins/test-data/41a388f8-1764-825e-e285-25964d37c9f3/.tmp/hbase.id 2024-11-12T14:43:19,887 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37395 is added to blk_1073741826_1002 (size=42) 2024-11-12T14:43:19,887 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36175 is added to blk_1073741826_1002 (size=42) 2024-11-12T14:43:19,888 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45975 is added to blk_1073741826_1002 (size=42) 2024-11-12T14:43:19,889 DEBUG [master/3945e1ca6f47:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:40559/user/jenkins/test-data/41a388f8-1764-825e-e285-25964d37c9f3/.tmp/hbase.id]:[hdfs://localhost:40559/user/jenkins/test-data/41a388f8-1764-825e-e285-25964d37c9f3/hbase.id] 2024-11-12T14:43:19,934 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-12T14:43:19,939 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-12T14:43:19,959 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 19ms. 2024-11-12T14:43:19,963 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38727-0x100346390eb0003, quorum=127.0.0.1:55341, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T14:43:19,963 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41431-0x100346390eb0001, quorum=127.0.0.1:55341, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T14:43:19,963 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33387-0x100346390eb0002, quorum=127.0.0.1:55341, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T14:43:19,963 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34233-0x100346390eb0000, quorum=127.0.0.1:55341, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T14:43:19,981 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36175 is added to blk_1073741827_1003 (size=196) 2024-11-12T14:43:19,981 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45975 is added to blk_1073741827_1003 (size=196) 2024-11-12T14:43:19,982 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37395 is added to blk_1073741827_1003 (size=196) 2024-11-12T14:43:19,997 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-12T14:43:19,999 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-12T14:43:20,003 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-12T14:43:20,035 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45975 is added to blk_1073741828_1004 (size=1189) 2024-11-12T14:43:20,036 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36175 is added to blk_1073741828_1004 (size=1189) 2024-11-12T14:43:20,036 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37395 is added to blk_1073741828_1004 (size=1189) 2024-11-12T14:43:20,055 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:40559/user/jenkins/test-data/41a388f8-1764-825e-e285-25964d37c9f3/MasterData/data/master/store 2024-11-12T14:43:20,077 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45975 is added to blk_1073741829_1005 (size=34) 2024-11-12T14:43:20,077 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36175 is added to blk_1073741829_1005 (size=34) 2024-11-12T14:43:20,078 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37395 is added to blk_1073741829_1005 (size=34) 2024-11-12T14:43:20,083 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-12T14:43:20,086 DEBUG [master/3945e1ca6f47:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T14:43:20,088 DEBUG [master/3945e1ca6f47:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-12T14:43:20,088 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-12T14:43:20,088 DEBUG [master/3945e1ca6f47:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-12T14:43:20,089 DEBUG [master/3945e1ca6f47:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-12T14:43:20,090 DEBUG [master/3945e1ca6f47:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-12T14:43:20,090 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-12T14:43:20,091 DEBUG [master/3945e1ca6f47:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731422600088Disabling compacts and flushes for region at 1731422600088Disabling writes for close at 1731422600089 (+1 ms)Writing region close event to WAL at 1731422600090 (+1 ms)Closed at 1731422600090 2024-11-12T14:43:20,092 WARN [master/3945e1ca6f47:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:40559/user/jenkins/test-data/41a388f8-1764-825e-e285-25964d37c9f3/MasterData/data/master/store/.initializing 2024-11-12T14:43:20,092 DEBUG [master/3945e1ca6f47:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:40559/user/jenkins/test-data/41a388f8-1764-825e-e285-25964d37c9f3/MasterData/WALs/3945e1ca6f47,34233,1731422599017 2024-11-12T14:43:20,099 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-12T14:43:20,114 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=3945e1ca6f47%2C34233%2C1731422599017, suffix=, logDir=hdfs://localhost:40559/user/jenkins/test-data/41a388f8-1764-825e-e285-25964d37c9f3/MasterData/WALs/3945e1ca6f47,34233,1731422599017, archiveDir=hdfs://localhost:40559/user/jenkins/test-data/41a388f8-1764-825e-e285-25964d37c9f3/MasterData/oldWALs, maxLogs=10 2024-11-12T14:43:20,142 DEBUG [master/3945e1ca6f47:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/41a388f8-1764-825e-e285-25964d37c9f3/MasterData/WALs/3945e1ca6f47,34233,1731422599017/3945e1ca6f47%2C34233%2C1731422599017.1731422600119, exclude list is [], retry=0 2024-11-12T14:43:20,162 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.initialize(FanOutOneBlockAsyncDFSOutputHelper.java:413) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$5.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:472) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$5.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:467) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.NettyFutureUtils.lambda$addListener$0(NettyFutureUtils.java:56) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setSuccess0(DefaultPromise.java:625) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.trySuccess(DefaultPromise.java:105) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPromise.trySuccess(DefaultChannelPromise.java:84) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:658) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:696) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:43:20,166 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36175,DS-31e7ba0a-e37c-4fe6-8d2a-a4846cefa408,DISK] 2024-11-12T14:43:20,166 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45975,DS-80b047b6-34d2-41c0-99c3-5cd61ca40075,DISK] 2024-11-12T14:43:20,166 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37395,DS-7317e4d9-cef6-4b85-a947-106c3c917da9,DISK] 2024-11-12T14:43:20,170 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-12T14:43:20,201 DEBUG [master/3945e1ca6f47:0:becomeActiveMaster {}] crypto.Encryption(565): Installed org.apache.hadoop.hbase.io.crypto.MockAesKeyProvider into key provider cache 2024-11-12T14:43:20,264 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/41a388f8-1764-825e-e285-25964d37c9f3/MasterData/WALs/3945e1ca6f47,34233,1731422599017/3945e1ca6f47%2C34233%2C1731422599017.1731422600119 2024-11-12T14:43:20,265 DEBUG [master/3945e1ca6f47:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40227:40227),(127.0.0.1/127.0.0.1:43643:43643),(127.0.0.1/127.0.0.1:36027:36027)] 2024-11-12T14:43:20,265 DEBUG [master/3945e1ca6f47:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-12T14:43:20,266 DEBUG [master/3945e1ca6f47:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T14:43:20,269 DEBUG [master/3945e1ca6f47:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-12T14:43:20,269 DEBUG [master/3945e1ca6f47:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-12T14:43:20,304 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-12T14:43:20,325 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-12T14:43:20,328 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:43:20,330 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-12T14:43:20,331 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-12T14:43:20,334 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-12T14:43:20,334 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:43:20,335 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T14:43:20,335 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-12T14:43:20,338 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-12T14:43:20,338 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:43:20,339 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T14:43:20,339 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-12T14:43:20,341 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-12T14:43:20,341 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:43:20,342 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T14:43:20,342 DEBUG [master/3945e1ca6f47:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-12T14:43:20,346 DEBUG [master/3945e1ca6f47:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40559/user/jenkins/test-data/41a388f8-1764-825e-e285-25964d37c9f3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-12T14:43:20,347 DEBUG [master/3945e1ca6f47:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40559/user/jenkins/test-data/41a388f8-1764-825e-e285-25964d37c9f3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-12T14:43:20,353 DEBUG [master/3945e1ca6f47:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-12T14:43:20,353 DEBUG [master/3945e1ca6f47:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-12T14:43:20,357 DEBUG [master/3945e1ca6f47:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-12T14:43:20,361 DEBUG [master/3945e1ca6f47:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-12T14:43:20,366 DEBUG [master/3945e1ca6f47:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40559/user/jenkins/test-data/41a388f8-1764-825e-e285-25964d37c9f3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-12T14:43:20,367 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71195098, jitterRate=0.060889631509780884}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-12T14:43:20,374 DEBUG [master/3945e1ca6f47:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731422600281Initializing all the Stores at 1731422600283 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731422600284 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731422600284Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731422600284Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731422600285 (+1 ms)Cleaning up temporary data from old regions at 1731422600353 (+68 ms)Region opened successfully at 1731422600374 (+21 ms) 2024-11-12T14:43:20,375 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-12T14:43:20,407 DEBUG [master/3945e1ca6f47:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@31e2207b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=3945e1ca6f47/172.17.0.2:0 2024-11-12T14:43:20,435 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-12T14:43:20,445 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-12T14:43:20,446 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-12T14:43:20,448 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-12T14:43:20,450 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-12T14:43:20,455 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-12T14:43:20,455 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-12T14:43:20,480 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-12T14:43:20,488 DEBUG [master/3945e1ca6f47:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34233-0x100346390eb0000, quorum=127.0.0.1:55341, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-12T14:43:20,490 DEBUG [master/3945e1ca6f47:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-12T14:43:20,493 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-12T14:43:20,495 DEBUG [master/3945e1ca6f47:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34233-0x100346390eb0000, quorum=127.0.0.1:55341, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-12T14:43:20,496 DEBUG [master/3945e1ca6f47:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-12T14:43:20,499 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-12T14:43:20,503 DEBUG [master/3945e1ca6f47:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34233-0x100346390eb0000, quorum=127.0.0.1:55341, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-12T14:43:20,505 DEBUG [master/3945e1ca6f47:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-12T14:43:20,506 DEBUG [master/3945e1ca6f47:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34233-0x100346390eb0000, quorum=127.0.0.1:55341, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-12T14:43:20,507 DEBUG [master/3945e1ca6f47:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-12T14:43:20,524 DEBUG [master/3945e1ca6f47:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34233-0x100346390eb0000, quorum=127.0.0.1:55341, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-12T14:43:20,525 DEBUG [master/3945e1ca6f47:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-12T14:43:20,528 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41431-0x100346390eb0001, quorum=127.0.0.1:55341, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-12T14:43:20,528 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33387-0x100346390eb0002, quorum=127.0.0.1:55341, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-12T14:43:20,529 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34233-0x100346390eb0000, quorum=127.0.0.1:55341, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-12T14:43:20,529 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38727-0x100346390eb0003, quorum=127.0.0.1:55341, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-12T14:43:20,529 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41431-0x100346390eb0001, quorum=127.0.0.1:55341, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T14:43:20,529 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38727-0x100346390eb0003, quorum=127.0.0.1:55341, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T14:43:20,529 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34233-0x100346390eb0000, quorum=127.0.0.1:55341, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T14:43:20,529 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33387-0x100346390eb0002, quorum=127.0.0.1:55341, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T14:43:20,532 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=3945e1ca6f47,34233,1731422599017, sessionid=0x100346390eb0000, setting cluster-up flag (Was=false) 2024-11-12T14:43:20,542 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38727-0x100346390eb0003, quorum=127.0.0.1:55341, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T14:43:20,542 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34233-0x100346390eb0000, quorum=127.0.0.1:55341, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T14:43:20,542 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41431-0x100346390eb0001, quorum=127.0.0.1:55341, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T14:43:20,542 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33387-0x100346390eb0002, quorum=127.0.0.1:55341, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T14:43:20,546 DEBUG [master/3945e1ca6f47:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-12T14:43:20,547 DEBUG [master/3945e1ca6f47:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=3945e1ca6f47,34233,1731422599017 2024-11-12T14:43:20,552 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33387-0x100346390eb0002, quorum=127.0.0.1:55341, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T14:43:20,552 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41431-0x100346390eb0001, quorum=127.0.0.1:55341, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T14:43:20,552 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34233-0x100346390eb0000, quorum=127.0.0.1:55341, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T14:43:20,552 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38727-0x100346390eb0003, quorum=127.0.0.1:55341, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T14:43:20,557 DEBUG [master/3945e1ca6f47:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-12T14:43:20,558 DEBUG [master/3945e1ca6f47:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=3945e1ca6f47,34233,1731422599017 2024-11-12T14:43:20,564 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:40559/user/jenkins/test-data/41a388f8-1764-825e-e285-25964d37c9f3/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-12T14:43:20,629 DEBUG [master/3945e1ca6f47:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-12T14:43:20,637 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-12T14:43:20,643 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-12T14:43:20,647 DEBUG [master/3945e1ca6f47:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 3945e1ca6f47,34233,1731422599017 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-12T14:43:20,654 DEBUG [master/3945e1ca6f47:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/3945e1ca6f47:0, corePoolSize=5, maxPoolSize=5 2024-11-12T14:43:20,654 DEBUG [master/3945e1ca6f47:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/3945e1ca6f47:0, corePoolSize=5, maxPoolSize=5 2024-11-12T14:43:20,654 DEBUG [master/3945e1ca6f47:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/3945e1ca6f47:0, corePoolSize=5, maxPoolSize=5 2024-11-12T14:43:20,654 DEBUG [master/3945e1ca6f47:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/3945e1ca6f47:0, corePoolSize=5, maxPoolSize=5 2024-11-12T14:43:20,654 DEBUG [master/3945e1ca6f47:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/3945e1ca6f47:0, corePoolSize=10, maxPoolSize=10 2024-11-12T14:43:20,654 DEBUG [master/3945e1ca6f47:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/3945e1ca6f47:0, corePoolSize=1, maxPoolSize=1 2024-11-12T14:43:20,655 DEBUG [master/3945e1ca6f47:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/3945e1ca6f47:0, corePoolSize=2, maxPoolSize=2 2024-11-12T14:43:20,655 DEBUG [master/3945e1ca6f47:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/3945e1ca6f47:0, corePoolSize=1, maxPoolSize=1 2024-11-12T14:43:20,659 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731422630659 2024-11-12T14:43:20,660 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-12T14:43:20,661 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-12T14:43:20,664 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-12T14:43:20,664 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-12T14:43:20,665 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-12T14:43:20,665 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-12T14:43:20,666 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-12T14:43:20,669 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-12T14:43:20,670 INFO [RS:2;3945e1ca6f47:38727 {}] regionserver.HRegionServer(746): ClusterId : 70468ada-4082-4450-9033-a07d7e00c036 2024-11-12T14:43:20,670 INFO [RS:0;3945e1ca6f47:41431 {}] regionserver.HRegionServer(746): ClusterId : 70468ada-4082-4450-9033-a07d7e00c036 2024-11-12T14:43:20,670 INFO [RS:1;3945e1ca6f47:33387 {}] regionserver.HRegionServer(746): ClusterId : 70468ada-4082-4450-9033-a07d7e00c036 2024-11-12T14:43:20,672 DEBUG [RS:2;3945e1ca6f47:38727 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-12T14:43:20,672 DEBUG [RS:0;3945e1ca6f47:41431 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-12T14:43:20,672 DEBUG [RS:1;3945e1ca6f47:33387 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-12T14:43:20,674 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:43:20,674 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-12T14:43:20,677 DEBUG [RS:2;3945e1ca6f47:38727 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-12T14:43:20,677 DEBUG [RS:0;3945e1ca6f47:41431 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-12T14:43:20,677 DEBUG [RS:2;3945e1ca6f47:38727 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-12T14:43:20,677 DEBUG [RS:0;3945e1ca6f47:41431 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-12T14:43:20,678 DEBUG [RS:1;3945e1ca6f47:33387 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-12T14:43:20,678 DEBUG [RS:1;3945e1ca6f47:33387 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-12T14:43:20,680 DEBUG [RS:0;3945e1ca6f47:41431 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-12T14:43:20,680 DEBUG [RS:2;3945e1ca6f47:38727 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-12T14:43:20,680 DEBUG [RS:1;3945e1ca6f47:33387 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-12T14:43:20,681 DEBUG [RS:0;3945e1ca6f47:41431 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3ae8eb14, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=3945e1ca6f47/172.17.0.2:0 2024-11-12T14:43:20,681 DEBUG [RS:2;3945e1ca6f47:38727 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@245285d5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=3945e1ca6f47/172.17.0.2:0 2024-11-12T14:43:20,681 DEBUG [RS:1;3945e1ca6f47:33387 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2c893397, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=3945e1ca6f47/172.17.0.2:0 2024-11-12T14:43:20,685 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-12T14:43:20,690 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-12T14:43:20,691 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-12T14:43:20,691 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-12T14:43:20,698 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-12T14:43:20,699 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-12T14:43:20,702 DEBUG [RS:2;3945e1ca6f47:38727 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:2;3945e1ca6f47:38727 2024-11-12T14:43:20,702 DEBUG [RS:0;3945e1ca6f47:41431 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;3945e1ca6f47:41431 2024-11-12T14:43:20,704 DEBUG [master/3945e1ca6f47:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/3945e1ca6f47:0:becomeActiveMaster-HFileCleaner.large.0-1731422600701,5,FailOnTimeoutGroup] 2024-11-12T14:43:20,705 DEBUG [RS:1;3945e1ca6f47:33387 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;3945e1ca6f47:33387 2024-11-12T14:43:20,706 DEBUG [master/3945e1ca6f47:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/3945e1ca6f47:0:becomeActiveMaster-HFileCleaner.small.0-1731422600704,5,FailOnTimeoutGroup] 2024-11-12T14:43:20,706 INFO [RS:2;3945e1ca6f47:38727 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-12T14:43:20,706 INFO [RS:0;3945e1ca6f47:41431 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-12T14:43:20,706 INFO [RS:1;3945e1ca6f47:33387 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-12T14:43:20,706 INFO [RS:2;3945e1ca6f47:38727 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-12T14:43:20,706 INFO [RS:1;3945e1ca6f47:33387 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-12T14:43:20,706 INFO [RS:0;3945e1ca6f47:41431 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-12T14:43:20,706 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-12T14:43:20,706 DEBUG [RS:2;3945e1ca6f47:38727 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-12T14:43:20,706 DEBUG [RS:1;3945e1ca6f47:33387 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-12T14:43:20,706 DEBUG [RS:0;3945e1ca6f47:41431 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-12T14:43:20,706 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-12T14:43:20,708 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-12T14:43:20,708 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-12T14:43:20,710 INFO [RS:1;3945e1ca6f47:33387 {}] regionserver.HRegionServer(2659): reportForDuty to master=3945e1ca6f47,34233,1731422599017 with port=33387, startcode=1731422599699 2024-11-12T14:43:20,710 INFO [RS:0;3945e1ca6f47:41431 {}] regionserver.HRegionServer(2659): reportForDuty to master=3945e1ca6f47,34233,1731422599017 with port=41431, startcode=1731422599613 2024-11-12T14:43:20,710 INFO [RS:2;3945e1ca6f47:38727 {}] regionserver.HRegionServer(2659): reportForDuty to master=3945e1ca6f47,34233,1731422599017 with port=38727, startcode=1731422599737 2024-11-12T14:43:20,713 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36175 is added to blk_1073741831_1007 (size=1321) 2024-11-12T14:43:20,713 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37395 is added to blk_1073741831_1007 (size=1321) 2024-11-12T14:43:20,715 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45975 is added to blk_1073741831_1007 (size=1321) 2024-11-12T14:43:20,716 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:40559/user/jenkins/test-data/41a388f8-1764-825e-e285-25964d37c9f3/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-12T14:43:20,717 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:40559/user/jenkins/test-data/41a388f8-1764-825e-e285-25964d37c9f3 2024-11-12T14:43:20,723 DEBUG [RS:0;3945e1ca6f47:41431 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-12T14:43:20,723 DEBUG [RS:2;3945e1ca6f47:38727 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-12T14:43:20,723 DEBUG [RS:1;3945e1ca6f47:33387 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-12T14:43:20,761 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36175 is added to blk_1073741832_1008 (size=32) 2024-11-12T14:43:20,761 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45975 is added to blk_1073741832_1008 (size=32) 2024-11-12T14:43:20,762 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37395 is added to blk_1073741832_1008 (size=32) 2024-11-12T14:43:20,765 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T14:43:20,767 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-12T14:43:20,770 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-12T14:43:20,771 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:43:20,772 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-12T14:43:20,772 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-12T14:43:20,775 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-12T14:43:20,775 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:43:20,776 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-12T14:43:20,777 INFO [HMaster-EventLoopGroup-2-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50991, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.2 (auth:SIMPLE), service=RegionServerStatusService 2024-11-12T14:43:20,777 INFO [HMaster-EventLoopGroup-2-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40631, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-12T14:43:20,777 INFO [HMaster-EventLoopGroup-2-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52249, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.1 (auth:SIMPLE), service=RegionServerStatusService 2024-11-12T14:43:20,777 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-12T14:43:20,780 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-12T14:43:20,780 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:43:20,781 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-12T14:43:20,782 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-12T14:43:20,784 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34233 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 3945e1ca6f47,41431,1731422599613 2024-11-12T14:43:20,785 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-12T14:43:20,785 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:43:20,786 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34233 {}] master.ServerManager(517): Registering regionserver=3945e1ca6f47,41431,1731422599613 2024-11-12T14:43:20,787 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-12T14:43:20,787 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-12T14:43:20,789 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40559/user/jenkins/test-data/41a388f8-1764-825e-e285-25964d37c9f3/data/hbase/meta/1588230740 2024-11-12T14:43:20,790 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40559/user/jenkins/test-data/41a388f8-1764-825e-e285-25964d37c9f3/data/hbase/meta/1588230740 2024-11-12T14:43:20,795 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-12T14:43:20,795 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-12T14:43:20,796 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-12T14:43:20,800 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34233 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 3945e1ca6f47,38727,1731422599737 2024-11-12T14:43:20,800 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-12T14:43:20,800 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34233 {}] master.ServerManager(517): Registering regionserver=3945e1ca6f47,38727,1731422599737 2024-11-12T14:43:20,803 DEBUG [RS:0;3945e1ca6f47:41431 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:40559/user/jenkins/test-data/41a388f8-1764-825e-e285-25964d37c9f3 2024-11-12T14:43:20,803 DEBUG [RS:0;3945e1ca6f47:41431 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:40559 2024-11-12T14:43:20,804 DEBUG [RS:0;3945e1ca6f47:41431 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-12T14:43:20,806 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40559/user/jenkins/test-data/41a388f8-1764-825e-e285-25964d37c9f3/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-12T14:43:20,807 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34233 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 3945e1ca6f47,33387,1731422599699 2024-11-12T14:43:20,807 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73438970, jitterRate=0.09432592988014221}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-12T14:43:20,808 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34233 {}] master.ServerManager(517): Registering regionserver=3945e1ca6f47,33387,1731422599699 2024-11-12T14:43:20,808 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34233-0x100346390eb0000, quorum=127.0.0.1:55341, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-12T14:43:20,808 DEBUG [RS:2;3945e1ca6f47:38727 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:40559/user/jenkins/test-data/41a388f8-1764-825e-e285-25964d37c9f3 2024-11-12T14:43:20,808 DEBUG [RS:2;3945e1ca6f47:38727 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:40559 2024-11-12T14:43:20,808 DEBUG [RS:2;3945e1ca6f47:38727 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-12T14:43:20,812 DEBUG [RS:1;3945e1ca6f47:33387 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:40559/user/jenkins/test-data/41a388f8-1764-825e-e285-25964d37c9f3 2024-11-12T14:43:20,812 DEBUG [RS:0;3945e1ca6f47:41431 {}] zookeeper.ZKUtil(111): regionserver:41431-0x100346390eb0001, quorum=127.0.0.1:55341, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/3945e1ca6f47,41431,1731422599613 2024-11-12T14:43:20,812 DEBUG [RS:1;3945e1ca6f47:33387 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:40559 2024-11-12T14:43:20,812 DEBUG [RS:1;3945e1ca6f47:33387 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-12T14:43:20,812 WARN [RS:0;3945e1ca6f47:41431 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-12T14:43:20,812 INFO [RS:0;3945e1ca6f47:41431 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-12T14:43:20,813 DEBUG [RS:0;3945e1ca6f47:41431 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:40559/user/jenkins/test-data/41a388f8-1764-825e-e285-25964d37c9f3/WALs/3945e1ca6f47,41431,1731422599613 2024-11-12T14:43:20,814 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [3945e1ca6f47,38727,1731422599737] 2024-11-12T14:43:20,814 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731422600765Initializing all the Stores at 1731422600767 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731422600767Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731422600767Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731422600767Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731422600767Cleaning up temporary data from old regions at 1731422600795 (+28 ms)Region opened successfully at 1731422600814 (+19 ms) 2024-11-12T14:43:20,814 DEBUG [RS:2;3945e1ca6f47:38727 {}] zookeeper.ZKUtil(111): regionserver:38727-0x100346390eb0003, quorum=127.0.0.1:55341, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/3945e1ca6f47,38727,1731422599737 2024-11-12T14:43:20,814 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [3945e1ca6f47,41431,1731422599613] 2024-11-12T14:43:20,814 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-12T14:43:20,814 WARN [RS:2;3945e1ca6f47:38727 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-12T14:43:20,814 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34233-0x100346390eb0000, quorum=127.0.0.1:55341, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-12T14:43:20,814 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-12T14:43:20,814 INFO [RS:2;3945e1ca6f47:38727 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-12T14:43:20,814 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-12T14:43:20,814 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-12T14:43:20,815 DEBUG [RS:2;3945e1ca6f47:38727 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:40559/user/jenkins/test-data/41a388f8-1764-825e-e285-25964d37c9f3/WALs/3945e1ca6f47,38727,1731422599737 2024-11-12T14:43:20,815 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-12T14:43:20,815 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [3945e1ca6f47,33387,1731422599699] 2024-11-12T14:43:20,816 DEBUG [RS:1;3945e1ca6f47:33387 {}] zookeeper.ZKUtil(111): regionserver:33387-0x100346390eb0002, quorum=127.0.0.1:55341, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/3945e1ca6f47,33387,1731422599699 2024-11-12T14:43:20,816 WARN [RS:1;3945e1ca6f47:33387 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-12T14:43:20,816 INFO [RS:1;3945e1ca6f47:33387 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-12T14:43:20,816 DEBUG [RS:1;3945e1ca6f47:33387 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:40559/user/jenkins/test-data/41a388f8-1764-825e-e285-25964d37c9f3/WALs/3945e1ca6f47,33387,1731422599699 2024-11-12T14:43:20,822 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-12T14:43:20,822 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731422600814Disabling compacts and flushes for region at 1731422600814Disabling writes for close at 1731422600815 (+1 ms)Writing region close event to WAL at 1731422600821 (+6 ms)Closed at 1731422600822 (+1 ms) 2024-11-12T14:43:20,826 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-12T14:43:20,826 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-12T14:43:20,833 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-12T14:43:20,844 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-12T14:43:20,848 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-12T14:43:20,849 INFO [RS:1;3945e1ca6f47:33387 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-12T14:43:20,849 INFO [RS:0;3945e1ca6f47:41431 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-12T14:43:20,849 INFO [RS:2;3945e1ca6f47:38727 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-12T14:43:20,865 INFO [RS:1;3945e1ca6f47:33387 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-12T14:43:20,865 INFO [RS:2;3945e1ca6f47:38727 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-12T14:43:20,865 INFO [RS:0;3945e1ca6f47:41431 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-12T14:43:20,869 INFO [RS:0;3945e1ca6f47:41431 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-12T14:43:20,869 INFO [RS:1;3945e1ca6f47:33387 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-12T14:43:20,870 INFO [RS:1;3945e1ca6f47:33387 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-12T14:43:20,870 INFO [RS:0;3945e1ca6f47:41431 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-12T14:43:20,873 INFO [RS:2;3945e1ca6f47:38727 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-12T14:43:20,873 INFO [RS:2;3945e1ca6f47:38727 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-12T14:43:20,874 INFO [RS:2;3945e1ca6f47:38727 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-12T14:43:20,874 INFO [RS:0;3945e1ca6f47:41431 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-12T14:43:20,874 INFO [RS:1;3945e1ca6f47:33387 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-12T14:43:20,880 INFO [RS:2;3945e1ca6f47:38727 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-12T14:43:20,880 INFO [RS:0;3945e1ca6f47:41431 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-12T14:43:20,880 INFO [RS:1;3945e1ca6f47:33387 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-12T14:43:20,881 INFO [RS:2;3945e1ca6f47:38727 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-12T14:43:20,881 INFO [RS:1;3945e1ca6f47:33387 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-12T14:43:20,881 INFO [RS:0;3945e1ca6f47:41431 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-12T14:43:20,882 DEBUG [RS:1;3945e1ca6f47:33387 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/3945e1ca6f47:0, corePoolSize=1, maxPoolSize=1 2024-11-12T14:43:20,882 DEBUG [RS:0;3945e1ca6f47:41431 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/3945e1ca6f47:0, corePoolSize=1, maxPoolSize=1 2024-11-12T14:43:20,882 DEBUG [RS:2;3945e1ca6f47:38727 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/3945e1ca6f47:0, corePoolSize=1, maxPoolSize=1 2024-11-12T14:43:20,882 DEBUG [RS:2;3945e1ca6f47:38727 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/3945e1ca6f47:0, corePoolSize=1, maxPoolSize=1 2024-11-12T14:43:20,882 DEBUG [RS:1;3945e1ca6f47:33387 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/3945e1ca6f47:0, corePoolSize=1, maxPoolSize=1 2024-11-12T14:43:20,882 DEBUG [RS:0;3945e1ca6f47:41431 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/3945e1ca6f47:0, corePoolSize=1, maxPoolSize=1 2024-11-12T14:43:20,882 DEBUG [RS:2;3945e1ca6f47:38727 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/3945e1ca6f47:0, corePoolSize=1, maxPoolSize=1 2024-11-12T14:43:20,882 DEBUG [RS:1;3945e1ca6f47:33387 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/3945e1ca6f47:0, corePoolSize=1, maxPoolSize=1 2024-11-12T14:43:20,882 DEBUG [RS:0;3945e1ca6f47:41431 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/3945e1ca6f47:0, corePoolSize=1, maxPoolSize=1 2024-11-12T14:43:20,882 DEBUG [RS:2;3945e1ca6f47:38727 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/3945e1ca6f47:0, corePoolSize=1, maxPoolSize=1 2024-11-12T14:43:20,882 DEBUG [RS:2;3945e1ca6f47:38727 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/3945e1ca6f47:0, corePoolSize=1, maxPoolSize=1 2024-11-12T14:43:20,882 DEBUG [RS:1;3945e1ca6f47:33387 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/3945e1ca6f47:0, corePoolSize=1, maxPoolSize=1 2024-11-12T14:43:20,882 DEBUG [RS:0;3945e1ca6f47:41431 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/3945e1ca6f47:0, corePoolSize=1, maxPoolSize=1 2024-11-12T14:43:20,882 DEBUG [RS:2;3945e1ca6f47:38727 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/3945e1ca6f47:0, corePoolSize=2, maxPoolSize=2 2024-11-12T14:43:20,882 DEBUG [RS:1;3945e1ca6f47:33387 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/3945e1ca6f47:0, corePoolSize=1, maxPoolSize=1 2024-11-12T14:43:20,882 DEBUG [RS:0;3945e1ca6f47:41431 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/3945e1ca6f47:0, corePoolSize=1, maxPoolSize=1 2024-11-12T14:43:20,882 DEBUG [RS:2;3945e1ca6f47:38727 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/3945e1ca6f47:0, corePoolSize=1, maxPoolSize=1 2024-11-12T14:43:20,882 DEBUG [RS:1;3945e1ca6f47:33387 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/3945e1ca6f47:0, corePoolSize=2, maxPoolSize=2 2024-11-12T14:43:20,882 DEBUG [RS:2;3945e1ca6f47:38727 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/3945e1ca6f47:0, corePoolSize=1, maxPoolSize=1 2024-11-12T14:43:20,882 DEBUG [RS:0;3945e1ca6f47:41431 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/3945e1ca6f47:0, corePoolSize=2, maxPoolSize=2 2024-11-12T14:43:20,883 DEBUG [RS:2;3945e1ca6f47:38727 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/3945e1ca6f47:0, corePoolSize=1, maxPoolSize=1 2024-11-12T14:43:20,883 DEBUG [RS:1;3945e1ca6f47:33387 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/3945e1ca6f47:0, corePoolSize=1, maxPoolSize=1 2024-11-12T14:43:20,883 DEBUG [RS:0;3945e1ca6f47:41431 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/3945e1ca6f47:0, corePoolSize=1, maxPoolSize=1 2024-11-12T14:43:20,883 DEBUG [RS:2;3945e1ca6f47:38727 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/3945e1ca6f47:0, corePoolSize=1, maxPoolSize=1 2024-11-12T14:43:20,883 DEBUG [RS:1;3945e1ca6f47:33387 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/3945e1ca6f47:0, corePoolSize=1, maxPoolSize=1 2024-11-12T14:43:20,883 DEBUG [RS:0;3945e1ca6f47:41431 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/3945e1ca6f47:0, corePoolSize=1, maxPoolSize=1 2024-11-12T14:43:20,883 DEBUG [RS:2;3945e1ca6f47:38727 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/3945e1ca6f47:0, corePoolSize=1, maxPoolSize=1 2024-11-12T14:43:20,883 DEBUG [RS:1;3945e1ca6f47:33387 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/3945e1ca6f47:0, corePoolSize=1, maxPoolSize=1 2024-11-12T14:43:20,883 DEBUG [RS:2;3945e1ca6f47:38727 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/3945e1ca6f47:0, corePoolSize=1, maxPoolSize=1 2024-11-12T14:43:20,883 DEBUG [RS:0;3945e1ca6f47:41431 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/3945e1ca6f47:0, corePoolSize=1, maxPoolSize=1 2024-11-12T14:43:20,883 DEBUG [RS:1;3945e1ca6f47:33387 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/3945e1ca6f47:0, corePoolSize=1, maxPoolSize=1 2024-11-12T14:43:20,883 DEBUG [RS:2;3945e1ca6f47:38727 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/3945e1ca6f47:0, corePoolSize=3, maxPoolSize=3 2024-11-12T14:43:20,883 DEBUG [RS:0;3945e1ca6f47:41431 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/3945e1ca6f47:0, corePoolSize=1, maxPoolSize=1 2024-11-12T14:43:20,883 DEBUG [RS:1;3945e1ca6f47:33387 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/3945e1ca6f47:0, corePoolSize=1, maxPoolSize=1 2024-11-12T14:43:20,883 DEBUG [RS:2;3945e1ca6f47:38727 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/3945e1ca6f47:0, corePoolSize=3, maxPoolSize=3 2024-11-12T14:43:20,883 DEBUG [RS:0;3945e1ca6f47:41431 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/3945e1ca6f47:0, corePoolSize=1, maxPoolSize=1 2024-11-12T14:43:20,883 DEBUG [RS:1;3945e1ca6f47:33387 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/3945e1ca6f47:0, corePoolSize=1, maxPoolSize=1 2024-11-12T14:43:20,883 DEBUG [RS:1;3945e1ca6f47:33387 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/3945e1ca6f47:0, corePoolSize=3, maxPoolSize=3 2024-11-12T14:43:20,883 DEBUG [RS:0;3945e1ca6f47:41431 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/3945e1ca6f47:0, corePoolSize=1, maxPoolSize=1 2024-11-12T14:43:20,883 DEBUG [RS:1;3945e1ca6f47:33387 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/3945e1ca6f47:0, corePoolSize=3, maxPoolSize=3 2024-11-12T14:43:20,883 DEBUG [RS:0;3945e1ca6f47:41431 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/3945e1ca6f47:0, corePoolSize=3, maxPoolSize=3 2024-11-12T14:43:20,884 DEBUG [RS:0;3945e1ca6f47:41431 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/3945e1ca6f47:0, corePoolSize=3, maxPoolSize=3 2024-11-12T14:43:20,885 INFO [RS:1;3945e1ca6f47:33387 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-12T14:43:20,885 INFO [RS:2;3945e1ca6f47:38727 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-12T14:43:20,885 INFO [RS:1;3945e1ca6f47:33387 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-12T14:43:20,885 INFO [RS:0;3945e1ca6f47:41431 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-12T14:43:20,885 INFO [RS:2;3945e1ca6f47:38727 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-12T14:43:20,885 INFO [RS:1;3945e1ca6f47:33387 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-12T14:43:20,885 INFO [RS:0;3945e1ca6f47:41431 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-12T14:43:20,885 INFO [RS:2;3945e1ca6f47:38727 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-12T14:43:20,885 INFO [RS:1;3945e1ca6f47:33387 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-12T14:43:20,885 INFO [RS:0;3945e1ca6f47:41431 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-12T14:43:20,885 INFO [RS:2;3945e1ca6f47:38727 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-12T14:43:20,885 INFO [RS:1;3945e1ca6f47:33387 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-12T14:43:20,885 INFO [RS:0;3945e1ca6f47:41431 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-12T14:43:20,885 INFO [RS:2;3945e1ca6f47:38727 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-12T14:43:20,885 INFO [RS:1;3945e1ca6f47:33387 {}] hbase.ChoreService(168): Chore ScheduledChore name=3945e1ca6f47,33387,1731422599699-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-12T14:43:20,885 INFO [RS:0;3945e1ca6f47:41431 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-12T14:43:20,885 INFO [RS:2;3945e1ca6f47:38727 {}] hbase.ChoreService(168): Chore ScheduledChore name=3945e1ca6f47,38727,1731422599737-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-12T14:43:20,885 INFO [RS:0;3945e1ca6f47:41431 {}] hbase.ChoreService(168): Chore ScheduledChore name=3945e1ca6f47,41431,1731422599613-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-12T14:43:20,911 INFO [RS:0;3945e1ca6f47:41431 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-12T14:43:20,911 INFO [RS:1;3945e1ca6f47:33387 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-12T14:43:20,911 INFO [RS:2;3945e1ca6f47:38727 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-12T14:43:20,913 INFO [RS:1;3945e1ca6f47:33387 {}] hbase.ChoreService(168): Chore ScheduledChore name=3945e1ca6f47,33387,1731422599699-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-12T14:43:20,913 INFO [RS:2;3945e1ca6f47:38727 {}] hbase.ChoreService(168): Chore ScheduledChore name=3945e1ca6f47,38727,1731422599737-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-12T14:43:20,913 INFO [RS:0;3945e1ca6f47:41431 {}] hbase.ChoreService(168): Chore ScheduledChore name=3945e1ca6f47,41431,1731422599613-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-12T14:43:20,913 INFO [RS:1;3945e1ca6f47:33387 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-12T14:43:20,913 INFO [RS:0;3945e1ca6f47:41431 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-12T14:43:20,913 INFO [RS:2;3945e1ca6f47:38727 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-12T14:43:20,913 INFO [RS:1;3945e1ca6f47:33387 {}] regionserver.Replication(171): 3945e1ca6f47,33387,1731422599699 started 2024-11-12T14:43:20,913 INFO [RS:2;3945e1ca6f47:38727 {}] regionserver.Replication(171): 3945e1ca6f47,38727,1731422599737 started 2024-11-12T14:43:20,913 INFO [RS:0;3945e1ca6f47:41431 {}] regionserver.Replication(171): 3945e1ca6f47,41431,1731422599613 started 2024-11-12T14:43:20,931 INFO [RS:1;3945e1ca6f47:33387 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-12T14:43:20,931 INFO [RS:1;3945e1ca6f47:33387 {}] regionserver.HRegionServer(1482): Serving as 3945e1ca6f47,33387,1731422599699, RpcServer on 3945e1ca6f47/172.17.0.2:33387, sessionid=0x100346390eb0002 2024-11-12T14:43:20,932 DEBUG [RS:1;3945e1ca6f47:33387 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-12T14:43:20,932 DEBUG [RS:1;3945e1ca6f47:33387 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 3945e1ca6f47,33387,1731422599699 2024-11-12T14:43:20,932 DEBUG [RS:1;3945e1ca6f47:33387 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '3945e1ca6f47,33387,1731422599699' 2024-11-12T14:43:20,932 DEBUG [RS:1;3945e1ca6f47:33387 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-12T14:43:20,933 DEBUG [RS:1;3945e1ca6f47:33387 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-12T14:43:20,934 DEBUG [RS:1;3945e1ca6f47:33387 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-12T14:43:20,934 DEBUG [RS:1;3945e1ca6f47:33387 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-12T14:43:20,934 DEBUG [RS:1;3945e1ca6f47:33387 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 3945e1ca6f47,33387,1731422599699 2024-11-12T14:43:20,934 DEBUG [RS:1;3945e1ca6f47:33387 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '3945e1ca6f47,33387,1731422599699' 2024-11-12T14:43:20,934 DEBUG [RS:1;3945e1ca6f47:33387 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-12T14:43:20,935 DEBUG [RS:1;3945e1ca6f47:33387 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-12T14:43:20,935 DEBUG [RS:1;3945e1ca6f47:33387 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-12T14:43:20,936 INFO [RS:1;3945e1ca6f47:33387 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-12T14:43:20,936 INFO [RS:1;3945e1ca6f47:33387 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-12T14:43:20,937 INFO [RS:0;3945e1ca6f47:41431 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-12T14:43:20,937 INFO [RS:2;3945e1ca6f47:38727 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-12T14:43:20,937 INFO [RS:2;3945e1ca6f47:38727 {}] regionserver.HRegionServer(1482): Serving as 3945e1ca6f47,38727,1731422599737, RpcServer on 3945e1ca6f47/172.17.0.2:38727, sessionid=0x100346390eb0003 2024-11-12T14:43:20,937 INFO [RS:0;3945e1ca6f47:41431 {}] regionserver.HRegionServer(1482): Serving as 3945e1ca6f47,41431,1731422599613, RpcServer on 3945e1ca6f47/172.17.0.2:41431, sessionid=0x100346390eb0001 2024-11-12T14:43:20,937 DEBUG [RS:2;3945e1ca6f47:38727 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-12T14:43:20,937 DEBUG [RS:0;3945e1ca6f47:41431 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-12T14:43:20,937 DEBUG [RS:0;3945e1ca6f47:41431 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 3945e1ca6f47,41431,1731422599613 2024-11-12T14:43:20,937 DEBUG [RS:2;3945e1ca6f47:38727 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 3945e1ca6f47,38727,1731422599737 2024-11-12T14:43:20,937 DEBUG [RS:2;3945e1ca6f47:38727 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '3945e1ca6f47,38727,1731422599737' 2024-11-12T14:43:20,937 DEBUG [RS:0;3945e1ca6f47:41431 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '3945e1ca6f47,41431,1731422599613' 2024-11-12T14:43:20,937 DEBUG [RS:2;3945e1ca6f47:38727 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-12T14:43:20,937 DEBUG [RS:0;3945e1ca6f47:41431 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-12T14:43:20,938 DEBUG [RS:2;3945e1ca6f47:38727 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-12T14:43:20,938 DEBUG [RS:0;3945e1ca6f47:41431 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-12T14:43:20,939 DEBUG [RS:2;3945e1ca6f47:38727 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-12T14:43:20,939 DEBUG [RS:0;3945e1ca6f47:41431 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-12T14:43:20,939 DEBUG [RS:0;3945e1ca6f47:41431 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-12T14:43:20,939 DEBUG [RS:2;3945e1ca6f47:38727 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-12T14:43:20,939 DEBUG [RS:2;3945e1ca6f47:38727 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 3945e1ca6f47,38727,1731422599737 2024-11-12T14:43:20,939 DEBUG [RS:0;3945e1ca6f47:41431 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 3945e1ca6f47,41431,1731422599613 2024-11-12T14:43:20,939 DEBUG [RS:2;3945e1ca6f47:38727 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '3945e1ca6f47,38727,1731422599737' 2024-11-12T14:43:20,939 DEBUG [RS:0;3945e1ca6f47:41431 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '3945e1ca6f47,41431,1731422599613' 2024-11-12T14:43:20,939 DEBUG [RS:2;3945e1ca6f47:38727 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-12T14:43:20,939 DEBUG [RS:0;3945e1ca6f47:41431 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-12T14:43:20,940 DEBUG [RS:2;3945e1ca6f47:38727 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-12T14:43:20,940 DEBUG [RS:0;3945e1ca6f47:41431 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-12T14:43:20,940 DEBUG [RS:0;3945e1ca6f47:41431 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-12T14:43:20,940 INFO [RS:0;3945e1ca6f47:41431 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-12T14:43:20,940 INFO [RS:0;3945e1ca6f47:41431 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-12T14:43:20,940 DEBUG [RS:2;3945e1ca6f47:38727 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-12T14:43:20,940 INFO [RS:2;3945e1ca6f47:38727 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-12T14:43:20,940 INFO [RS:2;3945e1ca6f47:38727 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-12T14:43:20,999 WARN [3945e1ca6f47:34233 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-12T14:43:21,042 INFO [RS:0;3945e1ca6f47:41431 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-12T14:43:21,042 INFO [RS:1;3945e1ca6f47:33387 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-12T14:43:21,042 INFO [RS:2;3945e1ca6f47:38727 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-12T14:43:21,045 INFO [RS:1;3945e1ca6f47:33387 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=3945e1ca6f47%2C33387%2C1731422599699, suffix=, logDir=hdfs://localhost:40559/user/jenkins/test-data/41a388f8-1764-825e-e285-25964d37c9f3/WALs/3945e1ca6f47,33387,1731422599699, archiveDir=hdfs://localhost:40559/user/jenkins/test-data/41a388f8-1764-825e-e285-25964d37c9f3/oldWALs, maxLogs=32 2024-11-12T14:43:21,045 INFO [RS:0;3945e1ca6f47:41431 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=3945e1ca6f47%2C41431%2C1731422599613, suffix=, logDir=hdfs://localhost:40559/user/jenkins/test-data/41a388f8-1764-825e-e285-25964d37c9f3/WALs/3945e1ca6f47,41431,1731422599613, archiveDir=hdfs://localhost:40559/user/jenkins/test-data/41a388f8-1764-825e-e285-25964d37c9f3/oldWALs, maxLogs=32 2024-11-12T14:43:21,045 INFO [RS:2;3945e1ca6f47:38727 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=3945e1ca6f47%2C38727%2C1731422599737, suffix=, logDir=hdfs://localhost:40559/user/jenkins/test-data/41a388f8-1764-825e-e285-25964d37c9f3/WALs/3945e1ca6f47,38727,1731422599737, archiveDir=hdfs://localhost:40559/user/jenkins/test-data/41a388f8-1764-825e-e285-25964d37c9f3/oldWALs, maxLogs=32 2024-11-12T14:43:21,062 DEBUG [RS:1;3945e1ca6f47:33387 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/41a388f8-1764-825e-e285-25964d37c9f3/WALs/3945e1ca6f47,33387,1731422599699/3945e1ca6f47%2C33387%2C1731422599699.1731422601049, exclude list is [], retry=0 2024-11-12T14:43:21,068 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37395,DS-7317e4d9-cef6-4b85-a947-106c3c917da9,DISK] 2024-11-12T14:43:21,068 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45975,DS-80b047b6-34d2-41c0-99c3-5cd61ca40075,DISK] 2024-11-12T14:43:21,068 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36175,DS-31e7ba0a-e37c-4fe6-8d2a-a4846cefa408,DISK] 2024-11-12T14:43:21,069 DEBUG [RS:2;3945e1ca6f47:38727 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/41a388f8-1764-825e-e285-25964d37c9f3/WALs/3945e1ca6f47,38727,1731422599737/3945e1ca6f47%2C38727%2C1731422599737.1731422601050, exclude list is [], retry=0 2024-11-12T14:43:21,069 DEBUG [RS:0;3945e1ca6f47:41431 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/41a388f8-1764-825e-e285-25964d37c9f3/WALs/3945e1ca6f47,41431,1731422599613/3945e1ca6f47%2C41431%2C1731422599613.1731422601050, exclude list is [], retry=0 2024-11-12T14:43:21,074 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36175,DS-31e7ba0a-e37c-4fe6-8d2a-a4846cefa408,DISK] 2024-11-12T14:43:21,074 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37395,DS-7317e4d9-cef6-4b85-a947-106c3c917da9,DISK] 2024-11-12T14:43:21,074 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36175,DS-31e7ba0a-e37c-4fe6-8d2a-a4846cefa408,DISK] 2024-11-12T14:43:21,074 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37395,DS-7317e4d9-cef6-4b85-a947-106c3c917da9,DISK] 2024-11-12T14:43:21,074 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45975,DS-80b047b6-34d2-41c0-99c3-5cd61ca40075,DISK] 2024-11-12T14:43:21,075 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45975,DS-80b047b6-34d2-41c0-99c3-5cd61ca40075,DISK] 2024-11-12T14:43:21,101 INFO [RS:1;3945e1ca6f47:33387 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/41a388f8-1764-825e-e285-25964d37c9f3/WALs/3945e1ca6f47,33387,1731422599699/3945e1ca6f47%2C33387%2C1731422599699.1731422601049 2024-11-12T14:43:21,105 DEBUG [RS:1;3945e1ca6f47:33387 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36027:36027),(127.0.0.1/127.0.0.1:43643:43643),(127.0.0.1/127.0.0.1:40227:40227)] 2024-11-12T14:43:21,111 INFO [RS:2;3945e1ca6f47:38727 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/41a388f8-1764-825e-e285-25964d37c9f3/WALs/3945e1ca6f47,38727,1731422599737/3945e1ca6f47%2C38727%2C1731422599737.1731422601050 2024-11-12T14:43:21,112 DEBUG [RS:2;3945e1ca6f47:38727 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40227:40227),(127.0.0.1/127.0.0.1:36027:36027),(127.0.0.1/127.0.0.1:43643:43643)] 2024-11-12T14:43:21,115 INFO [RS:0;3945e1ca6f47:41431 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/41a388f8-1764-825e-e285-25964d37c9f3/WALs/3945e1ca6f47,41431,1731422599613/3945e1ca6f47%2C41431%2C1731422599613.1731422601050 2024-11-12T14:43:21,115 DEBUG [RS:0;3945e1ca6f47:41431 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40227:40227),(127.0.0.1/127.0.0.1:36027:36027),(127.0.0.1/127.0.0.1:43643:43643)] 2024-11-12T14:43:21,253 DEBUG [3945e1ca6f47:34233 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=3, allServersCount=3 2024-11-12T14:43:21,263 DEBUG [3945e1ca6f47:34233 {}] balancer.BalancerClusterState(204): Hosts are {3945e1ca6f47=0} racks are {/default-rack=0} 2024-11-12T14:43:21,269 DEBUG [3945e1ca6f47:34233 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-12T14:43:21,269 DEBUG [3945e1ca6f47:34233 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-12T14:43:21,269 DEBUG [3945e1ca6f47:34233 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-11-12T14:43:21,269 DEBUG [3945e1ca6f47:34233 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-12T14:43:21,270 DEBUG [3945e1ca6f47:34233 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-12T14:43:21,270 DEBUG [3945e1ca6f47:34233 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-11-12T14:43:21,270 INFO [3945e1ca6f47:34233 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-12T14:43:21,270 INFO [3945e1ca6f47:34233 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-12T14:43:21,270 INFO [3945e1ca6f47:34233 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-11-12T14:43:21,270 DEBUG [3945e1ca6f47:34233 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-12T14:43:21,276 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=3945e1ca6f47,33387,1731422599699 2024-11-12T14:43:21,281 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 3945e1ca6f47,33387,1731422599699, state=OPENING 2024-11-12T14:43:21,285 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-12T14:43:21,286 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41431-0x100346390eb0001, quorum=127.0.0.1:55341, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T14:43:21,286 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38727-0x100346390eb0003, quorum=127.0.0.1:55341, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T14:43:21,286 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33387-0x100346390eb0002, quorum=127.0.0.1:55341, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T14:43:21,286 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34233-0x100346390eb0000, quorum=127.0.0.1:55341, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T14:43:21,287 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T14:43:21,287 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T14:43:21,287 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T14:43:21,287 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T14:43:21,288 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-12T14:43:21,290 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=3945e1ca6f47,33387,1731422599699}] 2024-11-12T14:43:21,462 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-12T14:43:21,465 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56247, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-12T14:43:21,476 INFO [RS_OPEN_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-12T14:43:21,477 INFO [RS_OPEN_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-12T14:43:21,477 INFO [RS_OPEN_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-12T14:43:21,481 INFO [RS_OPEN_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=3945e1ca6f47%2C33387%2C1731422599699.meta, suffix=.meta, logDir=hdfs://localhost:40559/user/jenkins/test-data/41a388f8-1764-825e-e285-25964d37c9f3/WALs/3945e1ca6f47,33387,1731422599699, archiveDir=hdfs://localhost:40559/user/jenkins/test-data/41a388f8-1764-825e-e285-25964d37c9f3/oldWALs, maxLogs=32 2024-11-12T14:43:21,498 DEBUG [RS_OPEN_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/41a388f8-1764-825e-e285-25964d37c9f3/WALs/3945e1ca6f47,33387,1731422599699/3945e1ca6f47%2C33387%2C1731422599699.meta.1731422601484.meta, exclude list is [], retry=0 2024-11-12T14:43:21,502 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36175,DS-31e7ba0a-e37c-4fe6-8d2a-a4846cefa408,DISK] 2024-11-12T14:43:21,502 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45975,DS-80b047b6-34d2-41c0-99c3-5cd61ca40075,DISK] 2024-11-12T14:43:21,502 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37395,DS-7317e4d9-cef6-4b85-a947-106c3c917da9,DISK] 2024-11-12T14:43:21,531 INFO [RS_OPEN_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/41a388f8-1764-825e-e285-25964d37c9f3/WALs/3945e1ca6f47,33387,1731422599699/3945e1ca6f47%2C33387%2C1731422599699.meta.1731422601484.meta 2024-11-12T14:43:21,532 DEBUG [RS_OPEN_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36027:36027),(127.0.0.1/127.0.0.1:43643:43643),(127.0.0.1/127.0.0.1:40227:40227)] 2024-11-12T14:43:21,532 DEBUG [RS_OPEN_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-12T14:43:21,534 DEBUG [RS_OPEN_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-12T14:43:21,536 DEBUG [RS_OPEN_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-12T14:43:21,540 INFO [RS_OPEN_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-12T14:43:21,544 DEBUG [RS_OPEN_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-12T14:43:21,544 DEBUG [RS_OPEN_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T14:43:21,545 DEBUG [RS_OPEN_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-12T14:43:21,545 DEBUG [RS_OPEN_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-12T14:43:21,548 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-12T14:43:21,550 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-12T14:43:21,550 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:43:21,551 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-12T14:43:21,551 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-12T14:43:21,552 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-12T14:43:21,552 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:43:21,553 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-12T14:43:21,554 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-12T14:43:21,555 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-12T14:43:21,555 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:43:21,556 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-12T14:43:21,556 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-12T14:43:21,557 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-12T14:43:21,557 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:43:21,558 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-12T14:43:21,558 DEBUG [RS_OPEN_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-12T14:43:21,560 DEBUG [RS_OPEN_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40559/user/jenkins/test-data/41a388f8-1764-825e-e285-25964d37c9f3/data/hbase/meta/1588230740 2024-11-12T14:43:21,562 DEBUG [RS_OPEN_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40559/user/jenkins/test-data/41a388f8-1764-825e-e285-25964d37c9f3/data/hbase/meta/1588230740 2024-11-12T14:43:21,564 DEBUG [RS_OPEN_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-12T14:43:21,564 DEBUG [RS_OPEN_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-12T14:43:21,565 DEBUG [RS_OPEN_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-12T14:43:21,567 DEBUG [RS_OPEN_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-12T14:43:21,569 INFO [RS_OPEN_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71876557, jitterRate=0.07104416191577911}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-12T14:43:21,569 DEBUG [RS_OPEN_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-12T14:43:21,571 DEBUG [RS_OPEN_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731422601545Writing region info on filesystem at 1731422601545Initializing all the Stores at 1731422601547 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731422601548 (+1 ms)Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731422601548Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731422601548Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731422601548Cleaning up temporary data from old regions at 1731422601564 (+16 ms)Running coprocessor post-open hooks at 1731422601569 (+5 ms)Region opened successfully at 1731422601570 (+1 ms) 2024-11-12T14:43:21,577 INFO [RS_OPEN_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731422601454 2024-11-12T14:43:21,588 DEBUG [RS_OPEN_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-12T14:43:21,589 INFO [RS_OPEN_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-12T14:43:21,591 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=3945e1ca6f47,33387,1731422599699 2024-11-12T14:43:21,593 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 3945e1ca6f47,33387,1731422599699, state=OPEN 2024-11-12T14:43:21,595 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38727-0x100346390eb0003, quorum=127.0.0.1:55341, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-12T14:43:21,595 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41431-0x100346390eb0001, quorum=127.0.0.1:55341, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-12T14:43:21,595 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34233-0x100346390eb0000, quorum=127.0.0.1:55341, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-12T14:43:21,595 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33387-0x100346390eb0002, quorum=127.0.0.1:55341, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-12T14:43:21,595 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T14:43:21,595 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T14:43:21,595 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T14:43:21,595 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T14:43:21,595 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=3945e1ca6f47,33387,1731422599699 2024-11-12T14:43:21,601 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-12T14:43:21,601 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=3945e1ca6f47,33387,1731422599699 in 306 msec 2024-11-12T14:43:21,608 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-12T14:43:21,608 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 770 msec 2024-11-12T14:43:21,610 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-12T14:43:21,610 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-12T14:43:21,627 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T14:43:21,628 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3945e1ca6f47,33387,1731422599699, seqNum=-1] 2024-11-12T14:43:21,645 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T14:43:21,648 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49095, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T14:43:21,667 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.0750 sec 2024-11-12T14:43:21,667 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731422601667, completionTime=-1 2024-11-12T14:43:21,670 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=3; waited=0ms, expected min=3 server(s), max=3 server(s), master is running 2024-11-12T14:43:21,670 DEBUG [master/3945e1ca6f47:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-12T14:43:21,709 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=3 2024-11-12T14:43:21,709 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731422661709 2024-11-12T14:43:21,710 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731422721710 2024-11-12T14:43:21,710 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 39 msec 2024-11-12T14:43:21,711 DEBUG [master/3945e1ca6f47:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-11-12T14:43:21,722 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=3945e1ca6f47,34233,1731422599017-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-12T14:43:21,723 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=3945e1ca6f47,34233,1731422599017-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-12T14:43:21,723 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=3945e1ca6f47,34233,1731422599017-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-12T14:43:21,724 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-3945e1ca6f47:34233, period=300000, unit=MILLISECONDS is enabled. 2024-11-12T14:43:21,724 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-12T14:43:21,725 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-12T14:43:21,731 DEBUG [master/3945e1ca6f47:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-12T14:43:21,749 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.934sec 2024-11-12T14:43:21,751 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-12T14:43:21,752 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-12T14:43:21,753 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-12T14:43:21,754 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-12T14:43:21,754 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-12T14:43:21,755 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=3945e1ca6f47,34233,1731422599017-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-12T14:43:21,755 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=3945e1ca6f47,34233,1731422599017-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-12T14:43:21,759 DEBUG [master/3945e1ca6f47:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-12T14:43:21,760 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-12T14:43:21,761 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=3945e1ca6f47,34233,1731422599017-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-12T14:43:21,775 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@154c1af6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T14:43:21,776 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3945e1ca6f47,34233,-1 for getting cluster id 2024-11-12T14:43:21,778 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T14:43:21,788 DEBUG [HMaster-EventLoopGroup-2-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '70468ada-4082-4450-9033-a07d7e00c036' 2024-11-12T14:43:21,790 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T14:43:21,790 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "70468ada-4082-4450-9033-a07d7e00c036" 2024-11-12T14:43:21,790 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@309895e0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T14:43:21,791 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3945e1ca6f47,34233,-1] 2024-11-12T14:43:21,793 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T14:43:21,795 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T14:43:21,796 INFO [HMaster-EventLoopGroup-2-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55050, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T14:43:21,799 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6581a620, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T14:43:21,799 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T14:43:21,807 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3945e1ca6f47,33387,1731422599699, seqNum=-1] 2024-11-12T14:43:21,807 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T14:43:21,810 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54642, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T14:43:21,829 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=3945e1ca6f47,34233,1731422599017 2024-11-12T14:43:21,829 INFO [Time-limited test {}] wal.AbstractTestWALReplay(147): hbase.rootdir=hdfs://localhost:40559/hbase 2024-11-12T14:43:21,843 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testNameConflictWhenSplit0 Thread=368, OpenFileDescriptor=607, MaxFileDescriptor=1048576, SystemLoadAverage=325, ProcessCount=11, AvailableMemoryMB=7810 2024-11-12T14:43:21,867 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-12T14:43:21,870 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-12T14:43:21,871 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-12T14:43:21,876 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-61660865, suffix=, logDir=hdfs://localhost:40559/hbase/WALs/hregion-61660865, archiveDir=hdfs://localhost:40559/hbase/oldWALs, maxLogs=32 2024-11-12T14:43:21,896 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-61660865/hregion-61660865.1731422601878, exclude list is [], retry=0 2024-11-12T14:43:21,901 DEBUG [AsyncFSWAL-8-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45975,DS-80b047b6-34d2-41c0-99c3-5cd61ca40075,DISK] 2024-11-12T14:43:21,901 DEBUG [AsyncFSWAL-8-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36175,DS-31e7ba0a-e37c-4fe6-8d2a-a4846cefa408,DISK] 2024-11-12T14:43:21,902 DEBUG [AsyncFSWAL-8-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37395,DS-7317e4d9-cef6-4b85-a947-106c3c917da9,DISK] 2024-11-12T14:43:21,929 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-61660865/hregion-61660865.1731422601878 2024-11-12T14:43:21,929 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43643:43643),(127.0.0.1/127.0.0.1:36027:36027),(127.0.0.1/127.0.0.1:40227:40227)] 2024-11-12T14:43:21,930 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 46f65e3bdf14090809c4018901f148a2, NAME => 'testReplayEditsWrittenIntoWAL,,1731422601868.46f65e3bdf14090809c4018901f148a2.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenIntoWAL', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40559/hbase 2024-11-12T14:43:21,941 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37395 is added to blk_1073741838_1014 (size=64) 2024-11-12T14:43:21,941 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45975 is added to blk_1073741838_1014 (size=64) 2024-11-12T14:43:21,941 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36175 is added to blk_1073741838_1014 (size=64) 2024-11-12T14:43:21,942 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1731422601868.46f65e3bdf14090809c4018901f148a2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T14:43:21,944 INFO [StoreOpener-46f65e3bdf14090809c4018901f148a2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 46f65e3bdf14090809c4018901f148a2 2024-11-12T14:43:21,946 INFO [StoreOpener-46f65e3bdf14090809c4018901f148a2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 46f65e3bdf14090809c4018901f148a2 columnFamilyName a 2024-11-12T14:43:21,946 DEBUG [StoreOpener-46f65e3bdf14090809c4018901f148a2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:43:21,947 INFO [StoreOpener-46f65e3bdf14090809c4018901f148a2-1 {}] regionserver.HStore(327): Store=46f65e3bdf14090809c4018901f148a2/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T14:43:21,948 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 46f65e3bdf14090809c4018901f148a2 2024-11-12T14:43:21,949 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40559/hbase/data/default/testReplayEditsWrittenIntoWAL/46f65e3bdf14090809c4018901f148a2 2024-11-12T14:43:21,950 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40559/hbase/data/default/testReplayEditsWrittenIntoWAL/46f65e3bdf14090809c4018901f148a2 2024-11-12T14:43:21,950 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 46f65e3bdf14090809c4018901f148a2 2024-11-12T14:43:21,950 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 46f65e3bdf14090809c4018901f148a2 2024-11-12T14:43:21,954 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 46f65e3bdf14090809c4018901f148a2 2024-11-12T14:43:21,958 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40559/hbase/data/default/testReplayEditsWrittenIntoWAL/46f65e3bdf14090809c4018901f148a2/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-12T14:43:21,959 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 46f65e3bdf14090809c4018901f148a2; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64801429, jitterRate=-0.03438346087932587}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-12T14:43:21,960 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 46f65e3bdf14090809c4018901f148a2: Writing region info on filesystem at 1731422601942Initializing all the Stores at 1731422601944 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731422601944Cleaning up temporary data from old regions at 1731422601950 (+6 ms)Region opened successfully at 1731422601960 (+10 ms) 2024-11-12T14:43:21,960 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 46f65e3bdf14090809c4018901f148a2, disabling compactions & flushes 2024-11-12T14:43:21,960 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1731422601868.46f65e3bdf14090809c4018901f148a2. 2024-11-12T14:43:21,960 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1731422601868.46f65e3bdf14090809c4018901f148a2. 2024-11-12T14:43:21,960 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1731422601868.46f65e3bdf14090809c4018901f148a2. after waiting 0 ms 2024-11-12T14:43:21,960 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1731422601868.46f65e3bdf14090809c4018901f148a2. 2024-11-12T14:43:21,960 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1731422601868.46f65e3bdf14090809c4018901f148a2. 2024-11-12T14:43:21,961 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 46f65e3bdf14090809c4018901f148a2: Waiting for close lock at 1731422601960Disabling compacts and flushes for region at 1731422601960Disabling writes for close at 1731422601960Writing region close event to WAL at 1731422601960Closed at 1731422601960 2024-11-12T14:43:21,965 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45975 is added to blk_1073741837_1013 (size=175) 2024-11-12T14:43:21,965 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37395 is added to blk_1073741837_1013 (size=175) 2024-11-12T14:43:21,966 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36175 is added to blk_1073741837_1013 (size=175) 2024-11-12T14:43:21,973 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-12T14:43:21,973 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-61660865:(num 1731422601878) 2024-11-12T14:43:21,975 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-11-12T14:43:22,013 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45975 is added to blk_1073741839_1015 (size=469) 2024-11-12T14:43:22,013 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36175 is added to blk_1073741839_1015 (size=469) 2024-11-12T14:43:22,014 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37395 is added to blk_1073741839_1015 (size=469) 2024-11-12T14:43:22,015 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-11-12T14:43:22,053 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45975 is added to blk_1073741840_1016 (size=322) 2024-11-12T14:43:22,053 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37395 is added to blk_1073741840_1016 (size=322) 2024-11-12T14:43:22,054 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36175 is added to blk_1073741840_1016 (size=322) 2024-11-12T14:43:22,075 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:40559/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731422601865/wal-1, size=469 (469bytes) 2024-11-12T14:43:22,076 DEBUG [Time-limited test {}] util.RecoverLeaseFSUtils(47): Initialize RecoverLeaseFSUtils 2024-11-12T14:43:22,076 DEBUG [Time-limited test {}] util.RecoverLeaseFSUtils(59): set recoverLeaseMethod to org.apache.hadoop.fs.LeaseRecoverable.recoverLease() 2024-11-12T14:43:22,076 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:40559/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731422601865/wal-1 2024-11-12T14:43:22,082 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:40559/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731422601865/wal-1 after 4ms 2024-11-12T14:43:22,112 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-12T14:43:22,112 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:40559/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731422601865/wal-1 took 38ms 2024-11-12T14:43:22,117 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:40559/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731422601865/wal-1 so closing down 2024-11-12T14:43:22,117 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-12T14:43:22,120 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal-1.temp 2024-11-12T14:43:22,146 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:40559/hbase/data/default/testReplayEditsWrittenIntoWAL/46f65e3bdf14090809c4018901f148a2/recovered.edits/0000000000000000001-wal-1.temp 2024-11-12T14:43:22,147 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-12T14:43:22,155 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37395 is added to blk_1073741841_1017 (size=469) 2024-11-12T14:43:22,155 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45975 is added to blk_1073741841_1017 (size=469) 2024-11-12T14:43:22,155 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36175 is added to blk_1073741841_1017 (size=469) 2024-11-12T14:43:22,159 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:40559/hbase/data/default/testReplayEditsWrittenIntoWAL/46f65e3bdf14090809c4018901f148a2/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) 2024-11-12T14:43:22,162 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:40559/hbase/data/default/testReplayEditsWrittenIntoWAL/46f65e3bdf14090809c4018901f148a2/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:40559/hbase/data/default/testReplayEditsWrittenIntoWAL/46f65e3bdf14090809c4018901f148a2/recovered.edits/0000000000000000002 2024-11-12T14:43:22,166 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 2 edits across 1 Regions in 49 ms; skipped=0; WAL=hdfs://localhost:40559/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731422601865/wal-1, size=469, length=469, corrupted=false, cancelled=false 2024-11-12T14:43:22,166 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:40559/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731422601865/wal-1, journal: Splitting hdfs://localhost:40559/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731422601865/wal-1, size=469 (469bytes) at 1731422602075Finishing writing output for hdfs://localhost:40559/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731422601865/wal-1 so closing down at 1731422602117 (+42 ms)Creating recovered edits writer path=hdfs://localhost:40559/hbase/data/default/testReplayEditsWrittenIntoWAL/46f65e3bdf14090809c4018901f148a2/recovered.edits/0000000000000000001-wal-1.temp at 1731422602146 (+29 ms)3 split writer threads finished at 1731422602147 (+1 ms)Closed recovered edits writer path=hdfs://localhost:40559/hbase/data/default/testReplayEditsWrittenIntoWAL/46f65e3bdf14090809c4018901f148a2/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) at 1731422602159 (+12 ms)Rename recovered edits hdfs://localhost:40559/hbase/data/default/testReplayEditsWrittenIntoWAL/46f65e3bdf14090809c4018901f148a2/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:40559/hbase/data/default/testReplayEditsWrittenIntoWAL/46f65e3bdf14090809c4018901f148a2/recovered.edits/0000000000000000002 at 1731422602162 (+3 ms)Processed 2 edits across 1 Regions in 49 ms; skipped=0; WAL=hdfs://localhost:40559/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731422601865/wal-1, size=469, length=469, corrupted=false, cancelled=false at 1731422602166 (+4 ms) 2024-11-12T14:43:22,181 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:40559/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731422601865/wal-2, size=322 (322bytes) 2024-11-12T14:43:22,181 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:40559/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731422601865/wal-2 2024-11-12T14:43:22,182 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:40559/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731422601865/wal-2 after 1ms 2024-11-12T14:43:22,210 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-12T14:43:22,210 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:40559/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731422601865/wal-2 took 29ms 2024-11-12T14:43:22,213 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:40559/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731422601865/wal-2 so closing down 2024-11-12T14:43:22,213 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-12T14:43:22,216 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000002-wal-2.temp 2024-11-12T14:43:22,241 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:40559/hbase/data/default/testReplayEditsWrittenIntoWAL/46f65e3bdf14090809c4018901f148a2/recovered.edits/0000000000000000002-wal-2.temp 2024-11-12T14:43:22,241 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-12T14:43:22,250 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37395 is added to blk_1073741842_1018 (size=322) 2024-11-12T14:43:22,250 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45975 is added to blk_1073741842_1018 (size=322) 2024-11-12T14:43:22,250 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36175 is added to blk_1073741842_1018 (size=322) 2024-11-12T14:43:22,251 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:40559/hbase/data/default/testReplayEditsWrittenIntoWAL/46f65e3bdf14090809c4018901f148a2/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-11-12T14:43:22,279 DEBUG [split-log-closeStream-pool-0 {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-12T14:43:22,282 WARN [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(184): Found existing old edits file and we have less entries. Deleting hdfs://localhost:40559/hbase/data/default/testReplayEditsWrittenIntoWAL/46f65e3bdf14090809c4018901f148a2/recovered.edits/0000000000000000002-wal-2.temp, length=322 2024-11-12T14:43:22,284 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 1 edits across 1 Regions in 74 ms; skipped=0; WAL=hdfs://localhost:40559/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731422601865/wal-2, size=322, length=322, corrupted=false, cancelled=false 2024-11-12T14:43:22,284 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:40559/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731422601865/wal-2, journal: Splitting hdfs://localhost:40559/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731422601865/wal-2, size=322 (322bytes) at 1731422602181Finishing writing output for hdfs://localhost:40559/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731422601865/wal-2 so closing down at 1731422602213 (+32 ms)Creating recovered edits writer path=hdfs://localhost:40559/hbase/data/default/testReplayEditsWrittenIntoWAL/46f65e3bdf14090809c4018901f148a2/recovered.edits/0000000000000000002-wal-2.temp at 1731422602241 (+28 ms)3 split writer threads finished at 1731422602241Closed recovered edits writer path=hdfs://localhost:40559/hbase/data/default/testReplayEditsWrittenIntoWAL/46f65e3bdf14090809c4018901f148a2/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1731422602251 (+10 ms)Processed 1 edits across 1 Regions in 74 ms; skipped=0; WAL=hdfs://localhost:40559/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731422601865/wal-2, size=322, length=322, corrupted=false, cancelled=false at 1731422602284 (+33 ms) 2024-11-12T14:43:22,284 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-12T14:43:22,286 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:40559/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731422601865, archiveDir=hdfs://localhost:40559/hbase/oldWALs, maxLogs=32 2024-11-12T14:43:22,300 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testnameconflictwhensplit0-manual,16010,1731422601865/wal.1731422602287, exclude list is [], retry=0 2024-11-12T14:43:22,304 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45975,DS-80b047b6-34d2-41c0-99c3-5cd61ca40075,DISK] 2024-11-12T14:43:22,305 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36175,DS-31e7ba0a-e37c-4fe6-8d2a-a4846cefa408,DISK] 2024-11-12T14:43:22,305 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37395,DS-7317e4d9-cef6-4b85-a947-106c3c917da9,DISK] 2024-11-12T14:43:22,330 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testnameconflictwhensplit0-manual,16010,1731422601865/wal.1731422602287 2024-11-12T14:43:22,330 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43643:43643),(127.0.0.1/127.0.0.1:36027:36027),(127.0.0.1/127.0.0.1:40227:40227)] 2024-11-12T14:43:22,330 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 46f65e3bdf14090809c4018901f148a2, NAME => 'testReplayEditsWrittenIntoWAL,,1731422601868.46f65e3bdf14090809c4018901f148a2.', STARTKEY => '', ENDKEY => ''} 2024-11-12T14:43:22,331 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1731422601868.46f65e3bdf14090809c4018901f148a2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T14:43:22,331 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 46f65e3bdf14090809c4018901f148a2 2024-11-12T14:43:22,331 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 46f65e3bdf14090809c4018901f148a2 2024-11-12T14:43:22,333 INFO [StoreOpener-46f65e3bdf14090809c4018901f148a2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 46f65e3bdf14090809c4018901f148a2 2024-11-12T14:43:22,334 INFO [StoreOpener-46f65e3bdf14090809c4018901f148a2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 46f65e3bdf14090809c4018901f148a2 columnFamilyName a 2024-11-12T14:43:22,334 DEBUG [StoreOpener-46f65e3bdf14090809c4018901f148a2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:43:22,335 INFO [StoreOpener-46f65e3bdf14090809c4018901f148a2-1 {}] regionserver.HStore(327): Store=46f65e3bdf14090809c4018901f148a2/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T14:43:22,335 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 46f65e3bdf14090809c4018901f148a2 2024-11-12T14:43:22,336 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40559/hbase/data/default/testReplayEditsWrittenIntoWAL/46f65e3bdf14090809c4018901f148a2 2024-11-12T14:43:22,339 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:40559/hbase/data/default/testReplayEditsWrittenIntoWAL/46f65e3bdf14090809c4018901f148a2 2024-11-12T14:43:22,341 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:40559/hbase/data/default/testReplayEditsWrittenIntoWAL/46f65e3bdf14090809c4018901f148a2/recovered.edits/0000000000000000002 2024-11-12T14:43:22,370 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-12T14:43:22,376 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 2, skipped 0, firstSequenceIdInLog=1, maxSequenceIdInLog=2, path=hdfs://localhost:40559/hbase/data/default/testReplayEditsWrittenIntoWAL/46f65e3bdf14090809c4018901f148a2/recovered.edits/0000000000000000002 2024-11-12T14:43:22,379 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 46f65e3bdf14090809c4018901f148a2 1/1 column families, dataSize=108 B heapSize=512 B 2024-11-12T14:43:22,426 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40559/hbase/data/default/testReplayEditsWrittenIntoWAL/46f65e3bdf14090809c4018901f148a2/.tmp/a/820f183ffc1b4c9a8e0287411bde4709 is 58, key is testReplayEditsWrittenIntoWAL/a:1/1731422601973/Put/seqid=0 2024-11-12T14:43:22,438 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45975 is added to blk_1073741844_1020 (size=5170) 2024-11-12T14:43:22,438 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37395 is added to blk_1073741844_1020 (size=5170) 2024-11-12T14:43:22,439 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36175 is added to blk_1073741844_1020 (size=5170) 2024-11-12T14:43:22,440 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=108 B at sequenceid=2 (bloomFilter=true), to=hdfs://localhost:40559/hbase/data/default/testReplayEditsWrittenIntoWAL/46f65e3bdf14090809c4018901f148a2/.tmp/a/820f183ffc1b4c9a8e0287411bde4709 2024-11-12T14:43:22,488 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40559/hbase/data/default/testReplayEditsWrittenIntoWAL/46f65e3bdf14090809c4018901f148a2/.tmp/a/820f183ffc1b4c9a8e0287411bde4709 as hdfs://localhost:40559/hbase/data/default/testReplayEditsWrittenIntoWAL/46f65e3bdf14090809c4018901f148a2/a/820f183ffc1b4c9a8e0287411bde4709 2024-11-12T14:43:22,505 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40559/hbase/data/default/testReplayEditsWrittenIntoWAL/46f65e3bdf14090809c4018901f148a2/a/820f183ffc1b4c9a8e0287411bde4709, entries=2, sequenceid=2, filesize=5.0 K 2024-11-12T14:43:22,513 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for 46f65e3bdf14090809c4018901f148a2 in 131ms, sequenceid=2, compaction requested=false; wal=null 2024-11-12T14:43:22,515 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:40559/hbase/data/default/testReplayEditsWrittenIntoWAL/46f65e3bdf14090809c4018901f148a2/recovered.edits/0000000000000000002 2024-11-12T14:43:22,515 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 46f65e3bdf14090809c4018901f148a2 2024-11-12T14:43:22,516 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 46f65e3bdf14090809c4018901f148a2 2024-11-12T14:43:22,521 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 46f65e3bdf14090809c4018901f148a2 2024-11-12T14:43:22,525 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40559/hbase/data/default/testReplayEditsWrittenIntoWAL/46f65e3bdf14090809c4018901f148a2/recovered.edits/2.seqid, newMaxSeqId=2, maxSeqId=1 2024-11-12T14:43:22,527 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 46f65e3bdf14090809c4018901f148a2; next sequenceid=3; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63143885, jitterRate=-0.059082791209220886}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-12T14:43:22,528 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 46f65e3bdf14090809c4018901f148a2: Writing region info on filesystem at 1731422602331Initializing all the Stores at 1731422602333 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731422602333Obtaining lock to block concurrent updates at 1731422602379 (+46 ms)Preparing flush snapshotting stores in 46f65e3bdf14090809c4018901f148a2 at 1731422602379Finished memstore snapshotting testReplayEditsWrittenIntoWAL,,1731422601868.46f65e3bdf14090809c4018901f148a2., syncing WAL and waiting on mvcc, flushsize=dataSize=108, getHeapSize=496, getOffHeapSize=0, getCellsCount=2 at 1731422602381 (+2 ms)Flushing stores of testReplayEditsWrittenIntoWAL,,1731422601868.46f65e3bdf14090809c4018901f148a2. at 1731422602381Flushing 46f65e3bdf14090809c4018901f148a2/a: creating writer at 1731422602382 (+1 ms)Flushing 46f65e3bdf14090809c4018901f148a2/a: appending metadata at 1731422602416 (+34 ms)Flushing 46f65e3bdf14090809c4018901f148a2/a: closing flushed file at 1731422602418 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3cadd751: reopening flushed file at 1731422602484 (+66 ms)Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for 46f65e3bdf14090809c4018901f148a2 in 131ms, sequenceid=2, compaction requested=false; wal=null at 1731422602513 (+29 ms)Cleaning up temporary data from old regions at 1731422602516 (+3 ms)Region opened successfully at 1731422602528 (+12 ms) 2024-11-12T14:43:22,555 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testNameConflictWhenSplit0 Thread=380 (was 368) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-425049348_22 at /127.0.0.1:45994 [Waiting for operation #13] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-8-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-425049348_22 at /127.0.0.1:47122 [Receiving block BP-1980882744-172.17.0.2-1731422596004:blk_1073741843_1019] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1980882744-172.17.0.2-1731422596004:blk_1073741843_1019, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1980882744-172.17.0.2-1731422596004:blk_1073741843_1019, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1980882744-172.17.0.2-1731422596004:blk_1073741843_1019, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: TestAsyncWALReplay-pool-0 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-425049348_22 at /127.0.0.1:46138 [Receiving block BP-1980882744-172.17.0.2-1731422596004:blk_1073741843_1019] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-8-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-0-hdfs://localhost:40559/hbase-prefix:default java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-425049348_22 at /127.0.0.1:50490 [Receiving block BP-1980882744-172.17.0.2-1731422596004:blk_1073741843_1019] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-425049348_22 at /127.0.0.1:47118 [Waiting for operation #5] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-425049348_22 at /127.0.0.1:50426 [Waiting for operation #4] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-8-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=703 (was 607) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=325 (was 325), ProcessCount=11 (was 11), AvailableMemoryMB=7798 (was 7810) 2024-11-12T14:43:22,565 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testNameConflictWhenSplit1 Thread=380, OpenFileDescriptor=703, MaxFileDescriptor=1048576, SystemLoadAverage=325, ProcessCount=11, AvailableMemoryMB=7798 2024-11-12T14:43:22,580 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-12T14:43:22,583 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-12T14:43:22,584 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-12T14:43:22,588 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-12933096, suffix=, logDir=hdfs://localhost:40559/hbase/WALs/hregion-12933096, archiveDir=hdfs://localhost:40559/hbase/oldWALs, maxLogs=32 2024-11-12T14:43:22,604 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-12933096/hregion-12933096.1731422602589, exclude list is [], retry=0 2024-11-12T14:43:22,608 DEBUG [AsyncFSWAL-10-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36175,DS-31e7ba0a-e37c-4fe6-8d2a-a4846cefa408,DISK] 2024-11-12T14:43:22,608 DEBUG [AsyncFSWAL-10-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45975,DS-80b047b6-34d2-41c0-99c3-5cd61ca40075,DISK] 2024-11-12T14:43:22,609 DEBUG [AsyncFSWAL-10-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37395,DS-7317e4d9-cef6-4b85-a947-106c3c917da9,DISK] 2024-11-12T14:43:22,637 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-12933096/hregion-12933096.1731422602589 2024-11-12T14:43:22,638 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36027:36027),(127.0.0.1/127.0.0.1:43643:43643),(127.0.0.1/127.0.0.1:40227:40227)] 2024-11-12T14:43:22,638 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 6b667ffc07b26ab289e91fd22d73f982, NAME => 'testReplayEditsWrittenIntoWAL,,1731422602581.6b667ffc07b26ab289e91fd22d73f982.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenIntoWAL', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40559/hbase 2024-11-12T14:43:22,656 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37395 is added to blk_1073741846_1022 (size=64) 2024-11-12T14:43:22,656 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45975 is added to blk_1073741846_1022 (size=64) 2024-11-12T14:43:22,656 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36175 is added to blk_1073741846_1022 (size=64) 2024-11-12T14:43:22,657 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1731422602581.6b667ffc07b26ab289e91fd22d73f982.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T14:43:22,659 INFO [StoreOpener-6b667ffc07b26ab289e91fd22d73f982-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 6b667ffc07b26ab289e91fd22d73f982 2024-11-12T14:43:22,661 INFO [StoreOpener-6b667ffc07b26ab289e91fd22d73f982-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6b667ffc07b26ab289e91fd22d73f982 columnFamilyName a 2024-11-12T14:43:22,661 DEBUG [StoreOpener-6b667ffc07b26ab289e91fd22d73f982-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:43:22,662 INFO [StoreOpener-6b667ffc07b26ab289e91fd22d73f982-1 {}] regionserver.HStore(327): Store=6b667ffc07b26ab289e91fd22d73f982/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T14:43:22,662 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 6b667ffc07b26ab289e91fd22d73f982 2024-11-12T14:43:22,663 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40559/hbase/data/default/testReplayEditsWrittenIntoWAL/6b667ffc07b26ab289e91fd22d73f982 2024-11-12T14:43:22,663 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40559/hbase/data/default/testReplayEditsWrittenIntoWAL/6b667ffc07b26ab289e91fd22d73f982 2024-11-12T14:43:22,664 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 6b667ffc07b26ab289e91fd22d73f982 2024-11-12T14:43:22,664 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 6b667ffc07b26ab289e91fd22d73f982 2024-11-12T14:43:22,667 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 6b667ffc07b26ab289e91fd22d73f982 2024-11-12T14:43:22,670 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40559/hbase/data/default/testReplayEditsWrittenIntoWAL/6b667ffc07b26ab289e91fd22d73f982/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-12T14:43:22,671 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 6b667ffc07b26ab289e91fd22d73f982; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68781348, jitterRate=0.024921953678131104}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-12T14:43:22,671 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 6b667ffc07b26ab289e91fd22d73f982: Writing region info on filesystem at 1731422602657Initializing all the Stores at 1731422602658 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731422602658Cleaning up temporary data from old regions at 1731422602664 (+6 ms)Region opened successfully at 1731422602671 (+7 ms) 2024-11-12T14:43:22,671 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 6b667ffc07b26ab289e91fd22d73f982, disabling compactions & flushes 2024-11-12T14:43:22,671 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1731422602581.6b667ffc07b26ab289e91fd22d73f982. 2024-11-12T14:43:22,671 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1731422602581.6b667ffc07b26ab289e91fd22d73f982. 2024-11-12T14:43:22,671 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1731422602581.6b667ffc07b26ab289e91fd22d73f982. after waiting 0 ms 2024-11-12T14:43:22,671 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1731422602581.6b667ffc07b26ab289e91fd22d73f982. 2024-11-12T14:43:22,672 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1731422602581.6b667ffc07b26ab289e91fd22d73f982. 2024-11-12T14:43:22,672 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 6b667ffc07b26ab289e91fd22d73f982: Waiting for close lock at 1731422602671Disabling compacts and flushes for region at 1731422602671Disabling writes for close at 1731422602671Writing region close event to WAL at 1731422602672 (+1 ms)Closed at 1731422602672 2024-11-12T14:43:22,676 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45975 is added to blk_1073741845_1021 (size=175) 2024-11-12T14:43:22,676 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36175 is added to blk_1073741845_1021 (size=175) 2024-11-12T14:43:22,676 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37395 is added to blk_1073741845_1021 (size=175) 2024-11-12T14:43:22,679 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-12T14:43:22,679 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-12933096:(num 1731422602589) 2024-11-12T14:43:22,680 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-11-12T14:43:22,716 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37395 is added to blk_1073741847_1023 (size=469) 2024-11-12T14:43:22,716 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36175 is added to blk_1073741847_1023 (size=469) 2024-11-12T14:43:22,716 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45975 is added to blk_1073741847_1023 (size=469) 2024-11-12T14:43:22,718 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-11-12T14:43:22,752 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45975 is added to blk_1073741848_1024 (size=322) 2024-11-12T14:43:22,753 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37395 is added to blk_1073741848_1024 (size=322) 2024-11-12T14:43:22,753 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36175 is added to blk_1073741848_1024 (size=322) 2024-11-12T14:43:22,768 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:40559/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731422602580/wal-2, size=322 (322bytes) 2024-11-12T14:43:22,768 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:40559/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731422602580/wal-2 2024-11-12T14:43:22,769 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:40559/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731422602580/wal-2 after 1ms 2024-11-12T14:43:22,795 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-12T14:43:22,796 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:40559/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731422602580/wal-2 took 28ms 2024-11-12T14:43:22,798 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:40559/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731422602580/wal-2 so closing down 2024-11-12T14:43:22,798 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-12T14:43:22,800 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000002-wal-2.temp 2024-11-12T14:43:22,827 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:40559/hbase/data/default/testReplayEditsWrittenIntoWAL/6b667ffc07b26ab289e91fd22d73f982/recovered.edits/0000000000000000002-wal-2.temp 2024-11-12T14:43:22,827 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-12T14:43:22,834 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36175 is added to blk_1073741849_1025 (size=322) 2024-11-12T14:43:22,834 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45975 is added to blk_1073741849_1025 (size=322) 2024-11-12T14:43:22,834 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37395 is added to blk_1073741849_1025 (size=322) 2024-11-12T14:43:22,835 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:40559/hbase/data/default/testReplayEditsWrittenIntoWAL/6b667ffc07b26ab289e91fd22d73f982/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-11-12T14:43:22,837 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:40559/hbase/data/default/testReplayEditsWrittenIntoWAL/6b667ffc07b26ab289e91fd22d73f982/recovered.edits/0000000000000000002-wal-2.temp to hdfs://localhost:40559/hbase/data/default/testReplayEditsWrittenIntoWAL/6b667ffc07b26ab289e91fd22d73f982/recovered.edits/0000000000000000002 2024-11-12T14:43:22,837 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 1 edits across 1 Regions in 41 ms; skipped=0; WAL=hdfs://localhost:40559/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731422602580/wal-2, size=322, length=322, corrupted=false, cancelled=false 2024-11-12T14:43:22,837 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:40559/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731422602580/wal-2, journal: Splitting hdfs://localhost:40559/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731422602580/wal-2, size=322 (322bytes) at 1731422602768Finishing writing output for hdfs://localhost:40559/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731422602580/wal-2 so closing down at 1731422602798 (+30 ms)Creating recovered edits writer path=hdfs://localhost:40559/hbase/data/default/testReplayEditsWrittenIntoWAL/6b667ffc07b26ab289e91fd22d73f982/recovered.edits/0000000000000000002-wal-2.temp at 1731422602827 (+29 ms)3 split writer threads finished at 1731422602827Closed recovered edits writer path=hdfs://localhost:40559/hbase/data/default/testReplayEditsWrittenIntoWAL/6b667ffc07b26ab289e91fd22d73f982/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1731422602835 (+8 ms)Rename recovered edits hdfs://localhost:40559/hbase/data/default/testReplayEditsWrittenIntoWAL/6b667ffc07b26ab289e91fd22d73f982/recovered.edits/0000000000000000002-wal-2.temp to hdfs://localhost:40559/hbase/data/default/testReplayEditsWrittenIntoWAL/6b667ffc07b26ab289e91fd22d73f982/recovered.edits/0000000000000000002 at 1731422602837 (+2 ms)Processed 1 edits across 1 Regions in 41 ms; skipped=0; WAL=hdfs://localhost:40559/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731422602580/wal-2, size=322, length=322, corrupted=false, cancelled=false at 1731422602837 2024-11-12T14:43:22,851 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:40559/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731422602580/wal-1, size=469 (469bytes) 2024-11-12T14:43:22,851 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:40559/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731422602580/wal-1 2024-11-12T14:43:22,852 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:40559/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731422602580/wal-1 after 1ms 2024-11-12T14:43:22,878 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-12T14:43:22,878 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:40559/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731422602580/wal-1 took 27ms 2024-11-12T14:43:22,881 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:40559/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731422602580/wal-1 so closing down 2024-11-12T14:43:22,881 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-12T14:43:22,883 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal-1.temp 2024-11-12T14:43:22,912 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:40559/hbase/data/default/testReplayEditsWrittenIntoWAL/6b667ffc07b26ab289e91fd22d73f982/recovered.edits/0000000000000000001-wal-1.temp 2024-11-12T14:43:22,913 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-12T14:43:22,920 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36175 is added to blk_1073741850_1026 (size=469) 2024-11-12T14:43:22,920 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37395 is added to blk_1073741850_1026 (size=469) 2024-11-12T14:43:22,920 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45975 is added to blk_1073741850_1026 (size=469) 2024-11-12T14:43:22,921 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:40559/hbase/data/default/testReplayEditsWrittenIntoWAL/6b667ffc07b26ab289e91fd22d73f982/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) 2024-11-12T14:43:22,948 DEBUG [split-log-closeStream-pool-0 {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-12T14:43:22,950 WARN [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(175): Found existing old edits file. It could be the result of a previous failed split attempt or we have duplicated wal entries. Deleting hdfs://localhost:40559/hbase/data/default/testReplayEditsWrittenIntoWAL/6b667ffc07b26ab289e91fd22d73f982/recovered.edits/0000000000000000002, length=322 2024-11-12T14:43:22,952 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:40559/hbase/data/default/testReplayEditsWrittenIntoWAL/6b667ffc07b26ab289e91fd22d73f982/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:40559/hbase/data/default/testReplayEditsWrittenIntoWAL/6b667ffc07b26ab289e91fd22d73f982/recovered.edits/0000000000000000002 2024-11-12T14:43:22,952 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 2 edits across 1 Regions in 73 ms; skipped=0; WAL=hdfs://localhost:40559/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731422602580/wal-1, size=469, length=469, corrupted=false, cancelled=false 2024-11-12T14:43:22,952 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:40559/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731422602580/wal-1, journal: Splitting hdfs://localhost:40559/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731422602580/wal-1, size=469 (469bytes) at 1731422602851Finishing writing output for hdfs://localhost:40559/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731422602580/wal-1 so closing down at 1731422602881 (+30 ms)Creating recovered edits writer path=hdfs://localhost:40559/hbase/data/default/testReplayEditsWrittenIntoWAL/6b667ffc07b26ab289e91fd22d73f982/recovered.edits/0000000000000000001-wal-1.temp at 1731422602912 (+31 ms)3 split writer threads finished at 1731422602913 (+1 ms)Closed recovered edits writer path=hdfs://localhost:40559/hbase/data/default/testReplayEditsWrittenIntoWAL/6b667ffc07b26ab289e91fd22d73f982/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) at 1731422602921 (+8 ms)Rename recovered edits hdfs://localhost:40559/hbase/data/default/testReplayEditsWrittenIntoWAL/6b667ffc07b26ab289e91fd22d73f982/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:40559/hbase/data/default/testReplayEditsWrittenIntoWAL/6b667ffc07b26ab289e91fd22d73f982/recovered.edits/0000000000000000002 at 1731422602952 (+31 ms)Processed 2 edits across 1 Regions in 73 ms; skipped=0; WAL=hdfs://localhost:40559/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731422602580/wal-1, size=469, length=469, corrupted=false, cancelled=false at 1731422602952 2024-11-12T14:43:22,953 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-12T14:43:22,955 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:40559/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731422602580, archiveDir=hdfs://localhost:40559/hbase/oldWALs, maxLogs=32 2024-11-12T14:43:22,969 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testnameconflictwhensplit1-manual,16010,1731422602580/wal.1731422602956, exclude list is [], retry=0 2024-11-12T14:43:22,973 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45975,DS-80b047b6-34d2-41c0-99c3-5cd61ca40075,DISK] 2024-11-12T14:43:22,973 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36175,DS-31e7ba0a-e37c-4fe6-8d2a-a4846cefa408,DISK] 2024-11-12T14:43:22,973 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37395,DS-7317e4d9-cef6-4b85-a947-106c3c917da9,DISK] 2024-11-12T14:43:23,000 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testnameconflictwhensplit1-manual,16010,1731422602580/wal.1731422602956 2024-11-12T14:43:23,001 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43643:43643),(127.0.0.1/127.0.0.1:36027:36027),(127.0.0.1/127.0.0.1:40227:40227)] 2024-11-12T14:43:23,001 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 6b667ffc07b26ab289e91fd22d73f982, NAME => 'testReplayEditsWrittenIntoWAL,,1731422602581.6b667ffc07b26ab289e91fd22d73f982.', STARTKEY => '', ENDKEY => ''} 2024-11-12T14:43:23,001 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1731422602581.6b667ffc07b26ab289e91fd22d73f982.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T14:43:23,001 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 6b667ffc07b26ab289e91fd22d73f982 2024-11-12T14:43:23,001 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 6b667ffc07b26ab289e91fd22d73f982 2024-11-12T14:43:23,003 INFO [StoreOpener-6b667ffc07b26ab289e91fd22d73f982-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 6b667ffc07b26ab289e91fd22d73f982 2024-11-12T14:43:23,005 INFO [StoreOpener-6b667ffc07b26ab289e91fd22d73f982-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6b667ffc07b26ab289e91fd22d73f982 columnFamilyName a 2024-11-12T14:43:23,005 DEBUG [StoreOpener-6b667ffc07b26ab289e91fd22d73f982-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:43:23,006 INFO [StoreOpener-6b667ffc07b26ab289e91fd22d73f982-1 {}] regionserver.HStore(327): Store=6b667ffc07b26ab289e91fd22d73f982/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T14:43:23,006 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 6b667ffc07b26ab289e91fd22d73f982 2024-11-12T14:43:23,007 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40559/hbase/data/default/testReplayEditsWrittenIntoWAL/6b667ffc07b26ab289e91fd22d73f982 2024-11-12T14:43:23,009 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:40559/hbase/data/default/testReplayEditsWrittenIntoWAL/6b667ffc07b26ab289e91fd22d73f982 2024-11-12T14:43:23,010 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:40559/hbase/data/default/testReplayEditsWrittenIntoWAL/6b667ffc07b26ab289e91fd22d73f982/recovered.edits/0000000000000000002 2024-11-12T14:43:23,038 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-12T14:43:23,040 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 2, skipped 0, firstSequenceIdInLog=1, maxSequenceIdInLog=2, path=hdfs://localhost:40559/hbase/data/default/testReplayEditsWrittenIntoWAL/6b667ffc07b26ab289e91fd22d73f982/recovered.edits/0000000000000000002 2024-11-12T14:43:23,040 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 6b667ffc07b26ab289e91fd22d73f982 1/1 column families, dataSize=108 B heapSize=512 B 2024-11-12T14:43:23,055 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40559/hbase/data/default/testReplayEditsWrittenIntoWAL/6b667ffc07b26ab289e91fd22d73f982/.tmp/a/31840e34936d4160954e71822233af88 is 58, key is testReplayEditsWrittenIntoWAL/a:1/1731422602679/Put/seqid=0 2024-11-12T14:43:23,066 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37395 is added to blk_1073741852_1028 (size=5170) 2024-11-12T14:43:23,067 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45975 is added to blk_1073741852_1028 (size=5170) 2024-11-12T14:43:23,067 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36175 is added to blk_1073741852_1028 (size=5170) 2024-11-12T14:43:23,068 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=108 B at sequenceid=2 (bloomFilter=true), to=hdfs://localhost:40559/hbase/data/default/testReplayEditsWrittenIntoWAL/6b667ffc07b26ab289e91fd22d73f982/.tmp/a/31840e34936d4160954e71822233af88 2024-11-12T14:43:23,077 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40559/hbase/data/default/testReplayEditsWrittenIntoWAL/6b667ffc07b26ab289e91fd22d73f982/.tmp/a/31840e34936d4160954e71822233af88 as hdfs://localhost:40559/hbase/data/default/testReplayEditsWrittenIntoWAL/6b667ffc07b26ab289e91fd22d73f982/a/31840e34936d4160954e71822233af88 2024-11-12T14:43:23,086 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40559/hbase/data/default/testReplayEditsWrittenIntoWAL/6b667ffc07b26ab289e91fd22d73f982/a/31840e34936d4160954e71822233af88, entries=2, sequenceid=2, filesize=5.0 K 2024-11-12T14:43:23,086 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for 6b667ffc07b26ab289e91fd22d73f982 in 46ms, sequenceid=2, compaction requested=false; wal=null 2024-11-12T14:43:23,087 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:40559/hbase/data/default/testReplayEditsWrittenIntoWAL/6b667ffc07b26ab289e91fd22d73f982/recovered.edits/0000000000000000002 2024-11-12T14:43:23,088 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 6b667ffc07b26ab289e91fd22d73f982 2024-11-12T14:43:23,088 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 6b667ffc07b26ab289e91fd22d73f982 2024-11-12T14:43:23,090 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 6b667ffc07b26ab289e91fd22d73f982 2024-11-12T14:43:23,093 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40559/hbase/data/default/testReplayEditsWrittenIntoWAL/6b667ffc07b26ab289e91fd22d73f982/recovered.edits/2.seqid, newMaxSeqId=2, maxSeqId=1 2024-11-12T14:43:23,094 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 6b667ffc07b26ab289e91fd22d73f982; next sequenceid=3; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66528588, jitterRate=-0.008646786212921143}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-12T14:43:23,094 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 6b667ffc07b26ab289e91fd22d73f982: Writing region info on filesystem at 1731422603001Initializing all the Stores at 1731422603003 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731422603003Obtaining lock to block concurrent updates at 1731422603040 (+37 ms)Preparing flush snapshotting stores in 6b667ffc07b26ab289e91fd22d73f982 at 1731422603040Finished memstore snapshotting testReplayEditsWrittenIntoWAL,,1731422602581.6b667ffc07b26ab289e91fd22d73f982., syncing WAL and waiting on mvcc, flushsize=dataSize=108, getHeapSize=496, getOffHeapSize=0, getCellsCount=2 at 1731422603040Flushing stores of testReplayEditsWrittenIntoWAL,,1731422602581.6b667ffc07b26ab289e91fd22d73f982. at 1731422603040Flushing 6b667ffc07b26ab289e91fd22d73f982/a: creating writer at 1731422603040Flushing 6b667ffc07b26ab289e91fd22d73f982/a: appending metadata at 1731422603054 (+14 ms)Flushing 6b667ffc07b26ab289e91fd22d73f982/a: closing flushed file at 1731422603054Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7858b088: reopening flushed file at 1731422603076 (+22 ms)Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for 6b667ffc07b26ab289e91fd22d73f982 in 46ms, sequenceid=2, compaction requested=false; wal=null at 1731422603086 (+10 ms)Cleaning up temporary data from old regions at 1731422603088 (+2 ms)Region opened successfully at 1731422603094 (+6 ms) 2024-11-12T14:43:23,111 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testNameConflictWhenSplit1 Thread=390 (was 380) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-425049348_22 at /127.0.0.1:46214 [Receiving block BP-1980882744-172.17.0.2-1731422596004:blk_1073741851_1027] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1980882744-172.17.0.2-1731422596004:blk_1073741851_1027, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1980882744-172.17.0.2-1731422596004:blk_1073741851_1027, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-425049348_22 at /127.0.0.1:47118 [Waiting for operation #12] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-10-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-10-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-425049348_22 at /127.0.0.1:45994 [Waiting for operation #19] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-425049348_22 at /127.0.0.1:47236 [Receiving block BP-1980882744-172.17.0.2-1731422596004:blk_1073741851_1027] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-425049348_22 at /127.0.0.1:50560 [Receiving block BP-1980882744-172.17.0.2-1731422596004:blk_1073741851_1027] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-10-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-425049348_22 at /127.0.0.1:50426 [Waiting for operation #6] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1980882744-172.17.0.2-1731422596004:blk_1073741851_1027, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=785 (was 703) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=325 (was 325), ProcessCount=11 (was 11), AvailableMemoryMB=7786 (was 7798) 2024-11-12T14:43:23,122 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsWrittenIntoWAL Thread=390, OpenFileDescriptor=785, MaxFileDescriptor=1048576, SystemLoadAverage=325, ProcessCount=11, AvailableMemoryMB=7785 2024-11-12T14:43:23,140 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-12T14:43:23,142 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-12T14:43:23,143 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-12T14:43:23,146 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-26135976, suffix=, logDir=hdfs://localhost:40559/hbase/WALs/hregion-26135976, archiveDir=hdfs://localhost:40559/hbase/oldWALs, maxLogs=32 2024-11-12T14:43:23,158 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-26135976/hregion-26135976.1731422603146, exclude list is [], retry=0 2024-11-12T14:43:23,161 DEBUG [AsyncFSWAL-12-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37395,DS-7317e4d9-cef6-4b85-a947-106c3c917da9,DISK] 2024-11-12T14:43:23,161 DEBUG [AsyncFSWAL-12-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45975,DS-80b047b6-34d2-41c0-99c3-5cd61ca40075,DISK] 2024-11-12T14:43:23,161 DEBUG [AsyncFSWAL-12-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36175,DS-31e7ba0a-e37c-4fe6-8d2a-a4846cefa408,DISK] 2024-11-12T14:43:23,185 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-26135976/hregion-26135976.1731422603146 2024-11-12T14:43:23,186 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40227:40227),(127.0.0.1/127.0.0.1:43643:43643),(127.0.0.1/127.0.0.1:36027:36027)] 2024-11-12T14:43:23,186 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 18980687202e69e67ba6536f50ecfecc, NAME => 'testReplayEditsWrittenIntoWAL,,1731422603141.18980687202e69e67ba6536f50ecfecc.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenIntoWAL', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40559/hbase 2024-11-12T14:43:23,198 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36175 is added to blk_1073741854_1030 (size=64) 2024-11-12T14:43:23,198 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37395 is added to blk_1073741854_1030 (size=64) 2024-11-12T14:43:23,198 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45975 is added to blk_1073741854_1030 (size=64) 2024-11-12T14:43:23,200 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1731422603141.18980687202e69e67ba6536f50ecfecc.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T14:43:23,202 INFO [StoreOpener-18980687202e69e67ba6536f50ecfecc-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 18980687202e69e67ba6536f50ecfecc 2024-11-12T14:43:23,204 INFO [StoreOpener-18980687202e69e67ba6536f50ecfecc-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 18980687202e69e67ba6536f50ecfecc columnFamilyName a 2024-11-12T14:43:23,204 DEBUG [StoreOpener-18980687202e69e67ba6536f50ecfecc-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:43:23,204 INFO [StoreOpener-18980687202e69e67ba6536f50ecfecc-1 {}] regionserver.HStore(327): Store=18980687202e69e67ba6536f50ecfecc/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T14:43:23,204 INFO [StoreOpener-18980687202e69e67ba6536f50ecfecc-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 18980687202e69e67ba6536f50ecfecc 2024-11-12T14:43:23,206 INFO [StoreOpener-18980687202e69e67ba6536f50ecfecc-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 18980687202e69e67ba6536f50ecfecc columnFamilyName b 2024-11-12T14:43:23,206 DEBUG [StoreOpener-18980687202e69e67ba6536f50ecfecc-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:43:23,207 INFO [StoreOpener-18980687202e69e67ba6536f50ecfecc-1 {}] regionserver.HStore(327): Store=18980687202e69e67ba6536f50ecfecc/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T14:43:23,207 INFO [StoreOpener-18980687202e69e67ba6536f50ecfecc-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 18980687202e69e67ba6536f50ecfecc 2024-11-12T14:43:23,209 INFO [StoreOpener-18980687202e69e67ba6536f50ecfecc-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 18980687202e69e67ba6536f50ecfecc columnFamilyName c 2024-11-12T14:43:23,209 DEBUG [StoreOpener-18980687202e69e67ba6536f50ecfecc-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:43:23,210 INFO [StoreOpener-18980687202e69e67ba6536f50ecfecc-1 {}] regionserver.HStore(327): Store=18980687202e69e67ba6536f50ecfecc/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T14:43:23,210 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 18980687202e69e67ba6536f50ecfecc 2024-11-12T14:43:23,211 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40559/hbase/data/default/testReplayEditsWrittenIntoWAL/18980687202e69e67ba6536f50ecfecc 2024-11-12T14:43:23,212 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40559/hbase/data/default/testReplayEditsWrittenIntoWAL/18980687202e69e67ba6536f50ecfecc 2024-11-12T14:43:23,213 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 18980687202e69e67ba6536f50ecfecc 2024-11-12T14:43:23,213 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 18980687202e69e67ba6536f50ecfecc 2024-11-12T14:43:23,214 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenIntoWAL descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-12T14:43:23,216 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 18980687202e69e67ba6536f50ecfecc 2024-11-12T14:43:23,219 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40559/hbase/data/default/testReplayEditsWrittenIntoWAL/18980687202e69e67ba6536f50ecfecc/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-12T14:43:23,219 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 18980687202e69e67ba6536f50ecfecc; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66366901, jitterRate=-0.011056110262870789}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-12T14:43:23,220 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 18980687202e69e67ba6536f50ecfecc: Writing region info on filesystem at 1731422603200Initializing all the Stores at 1731422603201 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731422603201Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731422603202 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731422603202Cleaning up temporary data from old regions at 1731422603213 (+11 ms)Region opened successfully at 1731422603219 (+6 ms) 2024-11-12T14:43:23,220 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 18980687202e69e67ba6536f50ecfecc, disabling compactions & flushes 2024-11-12T14:43:23,220 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1731422603141.18980687202e69e67ba6536f50ecfecc. 2024-11-12T14:43:23,220 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1731422603141.18980687202e69e67ba6536f50ecfecc. 2024-11-12T14:43:23,220 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1731422603141.18980687202e69e67ba6536f50ecfecc. after waiting 0 ms 2024-11-12T14:43:23,220 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1731422603141.18980687202e69e67ba6536f50ecfecc. 2024-11-12T14:43:23,220 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1731422603141.18980687202e69e67ba6536f50ecfecc. 2024-11-12T14:43:23,220 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 18980687202e69e67ba6536f50ecfecc: Waiting for close lock at 1731422603220Disabling compacts and flushes for region at 1731422603220Disabling writes for close at 1731422603220Writing region close event to WAL at 1731422603220Closed at 1731422603220 2024-11-12T14:43:23,223 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36175 is added to blk_1073741853_1029 (size=175) 2024-11-12T14:43:23,223 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45975 is added to blk_1073741853_1029 (size=175) 2024-11-12T14:43:23,223 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37395 is added to blk_1073741853_1029 (size=175) 2024-11-12T14:43:23,226 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-12T14:43:23,226 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-26135976:(num 1731422603146) 2024-11-12T14:43:23,226 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-12T14:43:23,228 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:40559/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731422603139, archiveDir=hdfs://localhost:40559/hbase/oldWALs, maxLogs=32 2024-11-12T14:43:23,244 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731422603139/wal.1731422603229, exclude list is [], retry=0 2024-11-12T14:43:23,247 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37395,DS-7317e4d9-cef6-4b85-a947-106c3c917da9,DISK] 2024-11-12T14:43:23,248 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45975,DS-80b047b6-34d2-41c0-99c3-5cd61ca40075,DISK] 2024-11-12T14:43:23,248 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36175,DS-31e7ba0a-e37c-4fe6-8d2a-a4846cefa408,DISK] 2024-11-12T14:43:23,274 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731422603139/wal.1731422603229 2024-11-12T14:43:23,274 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40227:40227),(127.0.0.1/127.0.0.1:43643:43643),(127.0.0.1/127.0.0.1:36027:36027)] 2024-11-12T14:43:23,496 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:40559/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731422603139/wal.1731422603229, size=0 (0bytes) 2024-11-12T14:43:23,496 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:40559/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731422603139/wal.1731422603229 might be still open, length is 0 2024-11-12T14:43:23,496 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:40559/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731422603139/wal.1731422603229 2024-11-12T14:43:23,497 WARN [IPC Server handler 2 on default port 40559 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731422603139/wal.1731422603229 has not been closed. Lease recovery is in progress. RecoveryId = 1032 for block blk_1073741855_1031 2024-11-12T14:43:23,497 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:40559/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731422603139/wal.1731422603229 after 1ms 2024-11-12T14:43:24,033 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-425049348_22 at /127.0.0.1:50576 [Receiving block BP-1980882744-172.17.0.2-1731422596004:blk_1073741855_1031] {}] datanode.DataXceiver(331): 127.0.0.1:37395:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:50576 dst: /127.0.0.1:37395 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:37395 remote=/127.0.0.1:50576]. Total timeout mills is 60000, 59433 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:43:24,035 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-425049348_22 at /127.0.0.1:47272 [Receiving block BP-1980882744-172.17.0.2-1731422596004:blk_1073741855_1031] {}] datanode.DataXceiver(331): 127.0.0.1:45975:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:47272 dst: /127.0.0.1:45975 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:43:24,035 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-425049348_22 at /127.0.0.1:46254 [Receiving block BP-1980882744-172.17.0.2-1731422596004:blk_1073741855_1031] {}] datanode.DataXceiver(331): 127.0.0.1:36175:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:46254 dst: /127.0.0.1:36175 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:43:24,050 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37395 is added to blk_1073741855_1032 (size=497775) 2024-11-12T14:43:24,052 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45975 is added to blk_1073741855_1032 (size=497775) 2024-11-12T14:43:24,055 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36175 is added to blk_1073741855_1032 (size=497775) 2024-11-12T14:43:27,049 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-12T14:43:27,119 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-12T14:43:27,498 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:40559/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731422603139/wal.1731422603229 after 4002ms 2024-11-12T14:43:27,529 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-12T14:43:27,531 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:40559/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731422603139/wal.1731422603229 took 4036ms 2024-11-12T14:43:27,536 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal.1731422603229.temp 2024-11-12T14:43:27,575 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:40559/hbase/data/default/testReplayEditsWrittenIntoWAL/18980687202e69e67ba6536f50ecfecc/recovered.edits/0000000000000000001-wal.1731422603229.temp 2024-11-12T14:43:27,632 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:40559/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731422603139/wal.1731422603229; continuing. 2024-11-12T14:43:27,632 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:40559/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731422603139/wal.1731422603229 so closing down 2024-11-12T14:43:27,632 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-12T14:43:27,633 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-12T14:43:27,640 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37395 is added to blk_1073741856_1033 (size=497783) 2024-11-12T14:43:27,640 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36175 is added to blk_1073741856_1033 (size=497783) 2024-11-12T14:43:27,641 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45975 is added to blk_1073741856_1033 (size=497783) 2024-11-12T14:43:27,643 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:40559/hbase/data/default/testReplayEditsWrittenIntoWAL/18980687202e69e67ba6536f50ecfecc/recovered.edits/0000000000000000001-wal.1731422603229.temp (wrote 3002 edits, skipped 0 edits in 50 ms) 2024-11-12T14:43:27,645 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:40559/hbase/data/default/testReplayEditsWrittenIntoWAL/18980687202e69e67ba6536f50ecfecc/recovered.edits/0000000000000000001-wal.1731422603229.temp to hdfs://localhost:40559/hbase/data/default/testReplayEditsWrittenIntoWAL/18980687202e69e67ba6536f50ecfecc/recovered.edits/0000000000000003002 2024-11-12T14:43:27,646 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 3002 edits across 1 Regions in 114 ms; skipped=0; WAL=hdfs://localhost:40559/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731422603139/wal.1731422603229, size=0, length=0, corrupted=false, cancelled=false 2024-11-12T14:43:27,646 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:40559/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731422603139/wal.1731422603229, journal: Splitting hdfs://localhost:40559/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731422603139/wal.1731422603229, size=0 (0bytes) at 1731422603496Creating recovered edits writer path=hdfs://localhost:40559/hbase/data/default/testReplayEditsWrittenIntoWAL/18980687202e69e67ba6536f50ecfecc/recovered.edits/0000000000000000001-wal.1731422603229.temp at 1731422607575 (+4079 ms)Split 1024 edits, skipped 0 edits. at 1731422607593 (+18 ms)Split 2048 edits, skipped 0 edits. at 1731422607615 (+22 ms)Finishing writing output for hdfs://localhost:40559/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731422603139/wal.1731422603229 so closing down at 1731422607632 (+17 ms)3 split writer threads finished at 1731422607633 (+1 ms)Closed recovered edits writer path=hdfs://localhost:40559/hbase/data/default/testReplayEditsWrittenIntoWAL/18980687202e69e67ba6536f50ecfecc/recovered.edits/0000000000000000001-wal.1731422603229.temp (wrote 3002 edits, skipped 0 edits in 50 ms) at 1731422607643 (+10 ms)Rename recovered edits hdfs://localhost:40559/hbase/data/default/testReplayEditsWrittenIntoWAL/18980687202e69e67ba6536f50ecfecc/recovered.edits/0000000000000000001-wal.1731422603229.temp to hdfs://localhost:40559/hbase/data/default/testReplayEditsWrittenIntoWAL/18980687202e69e67ba6536f50ecfecc/recovered.edits/0000000000000003002 at 1731422607645 (+2 ms)Processed 3002 edits across 1 Regions in 114 ms; skipped=0; WAL=hdfs://localhost:40559/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731422603139/wal.1731422603229, size=0, length=0, corrupted=false, cancelled=false at 1731422607646 (+1 ms) 2024-11-12T14:43:27,649 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:40559/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731422603139/wal.1731422603229 to hdfs://localhost:40559/hbase/oldWALs/wal.1731422603229 2024-11-12T14:43:27,650 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:40559/hbase/data/default/testReplayEditsWrittenIntoWAL/18980687202e69e67ba6536f50ecfecc/recovered.edits/0000000000000003002 2024-11-12T14:43:27,651 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-12T14:43:27,653 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:40559/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731422603139, archiveDir=hdfs://localhost:40559/hbase/oldWALs, maxLogs=32 2024-11-12T14:43:27,667 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731422603139/wal.1731422607653, exclude list is [], retry=0 2024-11-12T14:43:27,670 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45975,DS-80b047b6-34d2-41c0-99c3-5cd61ca40075,DISK] 2024-11-12T14:43:27,671 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37395,DS-7317e4d9-cef6-4b85-a947-106c3c917da9,DISK] 2024-11-12T14:43:27,671 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36175,DS-31e7ba0a-e37c-4fe6-8d2a-a4846cefa408,DISK] 2024-11-12T14:43:27,698 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731422603139/wal.1731422607653 2024-11-12T14:43:27,699 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43643:43643),(127.0.0.1/127.0.0.1:40227:40227),(127.0.0.1/127.0.0.1:36027:36027)] 2024-11-12T14:43:27,699 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1731422603141.18980687202e69e67ba6536f50ecfecc.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T14:43:27,701 INFO [StoreOpener-18980687202e69e67ba6536f50ecfecc-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 18980687202e69e67ba6536f50ecfecc 2024-11-12T14:43:27,703 INFO [StoreOpener-18980687202e69e67ba6536f50ecfecc-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 18980687202e69e67ba6536f50ecfecc columnFamilyName a 2024-11-12T14:43:27,703 DEBUG [StoreOpener-18980687202e69e67ba6536f50ecfecc-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:43:27,703 INFO [StoreOpener-18980687202e69e67ba6536f50ecfecc-1 {}] regionserver.HStore(327): Store=18980687202e69e67ba6536f50ecfecc/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T14:43:27,704 INFO [StoreOpener-18980687202e69e67ba6536f50ecfecc-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 18980687202e69e67ba6536f50ecfecc 2024-11-12T14:43:27,705 INFO [StoreOpener-18980687202e69e67ba6536f50ecfecc-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 18980687202e69e67ba6536f50ecfecc columnFamilyName b 2024-11-12T14:43:27,705 DEBUG [StoreOpener-18980687202e69e67ba6536f50ecfecc-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:43:27,706 INFO [StoreOpener-18980687202e69e67ba6536f50ecfecc-1 {}] regionserver.HStore(327): Store=18980687202e69e67ba6536f50ecfecc/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T14:43:27,706 INFO [StoreOpener-18980687202e69e67ba6536f50ecfecc-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 18980687202e69e67ba6536f50ecfecc 2024-11-12T14:43:27,707 INFO [StoreOpener-18980687202e69e67ba6536f50ecfecc-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 18980687202e69e67ba6536f50ecfecc columnFamilyName c 2024-11-12T14:43:27,707 DEBUG [StoreOpener-18980687202e69e67ba6536f50ecfecc-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:43:27,708 INFO [StoreOpener-18980687202e69e67ba6536f50ecfecc-1 {}] regionserver.HStore(327): Store=18980687202e69e67ba6536f50ecfecc/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T14:43:27,708 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 18980687202e69e67ba6536f50ecfecc 2024-11-12T14:43:27,709 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40559/hbase/data/default/testReplayEditsWrittenIntoWAL/18980687202e69e67ba6536f50ecfecc 2024-11-12T14:43:27,711 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:40559/hbase/data/default/testReplayEditsWrittenIntoWAL/18980687202e69e67ba6536f50ecfecc 2024-11-12T14:43:27,712 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:40559/hbase/data/default/testReplayEditsWrittenIntoWAL/18980687202e69e67ba6536f50ecfecc/recovered.edits/0000000000000003002 2024-11-12T14:43:27,740 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-12T14:43:27,786 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-11-12T14:43:28,135 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 18980687202e69e67ba6536f50ecfecc 3/3 column families, dataSize=42.49 KB heapSize=100.11 KB 2024-11-12T14:43:28,174 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40559/hbase/data/default/testReplayEditsWrittenIntoWAL/18980687202e69e67ba6536f50ecfecc/.tmp/a/1587b6c86ca34ecc89d16134c903fbe5 is 62, key is testReplayEditsWrittenIntoWAL/a:100/1731422603285/Put/seqid=0 2024-11-12T14:43:28,184 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45975 is added to blk_1073741858_1035 (size=50463) 2024-11-12T14:43:28,184 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37395 is added to blk_1073741858_1035 (size=50463) 2024-11-12T14:43:28,184 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36175 is added to blk_1073741858_1035 (size=50463) 2024-11-12T14:43:28,185 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=754 (bloomFilter=true), to=hdfs://localhost:40559/hbase/data/default/testReplayEditsWrittenIntoWAL/18980687202e69e67ba6536f50ecfecc/.tmp/a/1587b6c86ca34ecc89d16134c903fbe5 2024-11-12T14:43:28,194 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40559/hbase/data/default/testReplayEditsWrittenIntoWAL/18980687202e69e67ba6536f50ecfecc/.tmp/a/1587b6c86ca34ecc89d16134c903fbe5 as hdfs://localhost:40559/hbase/data/default/testReplayEditsWrittenIntoWAL/18980687202e69e67ba6536f50ecfecc/a/1587b6c86ca34ecc89d16134c903fbe5 2024-11-12T14:43:28,202 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40559/hbase/data/default/testReplayEditsWrittenIntoWAL/18980687202e69e67ba6536f50ecfecc/a/1587b6c86ca34ecc89d16134c903fbe5, entries=754, sequenceid=754, filesize=49.3 K 2024-11-12T14:43:28,202 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~42.49 KB/43512, heapSize ~99.59 KB/101984, currentSize=0 B/0 for 18980687202e69e67ba6536f50ecfecc in 67ms, sequenceid=754, compaction requested=false; wal=null 2024-11-12T14:43:28,222 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-11-12T14:43:28,222 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 18980687202e69e67ba6536f50ecfecc 3/3 column families, dataSize=42.49 KB heapSize=100.11 KB 2024-11-12T14:43:28,231 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40559/hbase/data/default/testReplayEditsWrittenIntoWAL/18980687202e69e67ba6536f50ecfecc/.tmp/a/f51cb30d6a46464aa353f21e45850598 is 62, key is testReplayEditsWrittenIntoWAL/a:754/1731422603340/Put/seqid=0 2024-11-12T14:43:28,238 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36175 is added to blk_1073741859_1036 (size=20072) 2024-11-12T14:43:28,238 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45975 is added to blk_1073741859_1036 (size=20072) 2024-11-12T14:43:28,239 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37395 is added to blk_1073741859_1036 (size=20072) 2024-11-12T14:43:28,239 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.93 KB at sequenceid=1508 (bloomFilter=true), to=hdfs://localhost:40559/hbase/data/default/testReplayEditsWrittenIntoWAL/18980687202e69e67ba6536f50ecfecc/.tmp/a/f51cb30d6a46464aa353f21e45850598 2024-11-12T14:43:28,263 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40559/hbase/data/default/testReplayEditsWrittenIntoWAL/18980687202e69e67ba6536f50ecfecc/.tmp/b/6f4862fc5ff84fe7bb3a0c4e7586c702 is 62, key is testReplayEditsWrittenIntoWAL/b:100/1731422603364/Put/seqid=0 2024-11-12T14:43:28,270 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37395 is added to blk_1073741860_1037 (size=35835) 2024-11-12T14:43:28,270 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36175 is added to blk_1073741860_1037 (size=35835) 2024-11-12T14:43:28,271 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45975 is added to blk_1073741860_1037 (size=35835) 2024-11-12T14:43:28,271 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=28.56 KB at sequenceid=1508 (bloomFilter=true), to=hdfs://localhost:40559/hbase/data/default/testReplayEditsWrittenIntoWAL/18980687202e69e67ba6536f50ecfecc/.tmp/b/6f4862fc5ff84fe7bb3a0c4e7586c702 2024-11-12T14:43:28,279 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40559/hbase/data/default/testReplayEditsWrittenIntoWAL/18980687202e69e67ba6536f50ecfecc/.tmp/a/f51cb30d6a46464aa353f21e45850598 as hdfs://localhost:40559/hbase/data/default/testReplayEditsWrittenIntoWAL/18980687202e69e67ba6536f50ecfecc/a/f51cb30d6a46464aa353f21e45850598 2024-11-12T14:43:28,286 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40559/hbase/data/default/testReplayEditsWrittenIntoWAL/18980687202e69e67ba6536f50ecfecc/a/f51cb30d6a46464aa353f21e45850598, entries=246, sequenceid=1508, filesize=19.6 K 2024-11-12T14:43:28,288 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40559/hbase/data/default/testReplayEditsWrittenIntoWAL/18980687202e69e67ba6536f50ecfecc/.tmp/b/6f4862fc5ff84fe7bb3a0c4e7586c702 as hdfs://localhost:40559/hbase/data/default/testReplayEditsWrittenIntoWAL/18980687202e69e67ba6536f50ecfecc/b/6f4862fc5ff84fe7bb3a0c4e7586c702 2024-11-12T14:43:28,295 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40559/hbase/data/default/testReplayEditsWrittenIntoWAL/18980687202e69e67ba6536f50ecfecc/b/6f4862fc5ff84fe7bb3a0c4e7586c702, entries=508, sequenceid=1508, filesize=35.0 K 2024-11-12T14:43:28,295 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~42.49 KB/43512, heapSize ~99.83 KB/102224, currentSize=0 B/0 for 18980687202e69e67ba6536f50ecfecc in 73ms, sequenceid=1508, compaction requested=false; wal=null 2024-11-12T14:43:28,307 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-11-12T14:43:28,307 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 18980687202e69e67ba6536f50ecfecc 3/3 column families, dataSize=42.49 KB heapSize=100.11 KB 2024-11-12T14:43:28,314 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40559/hbase/data/default/testReplayEditsWrittenIntoWAL/18980687202e69e67ba6536f50ecfecc/.tmp/b/d884bbd1c7ec4a5684c0f96c48550697 is 62, key is testReplayEditsWrittenIntoWAL/b:508/1731422603386/Put/seqid=0 2024-11-12T14:43:28,324 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37395 is added to blk_1073741861_1038 (size=35082) 2024-11-12T14:43:28,325 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36175 is added to blk_1073741861_1038 (size=35082) 2024-11-12T14:43:28,325 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45975 is added to blk_1073741861_1038 (size=35082) 2024-11-12T14:43:28,326 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=27.87 KB at sequenceid=2262 (bloomFilter=true), to=hdfs://localhost:40559/hbase/data/default/testReplayEditsWrittenIntoWAL/18980687202e69e67ba6536f50ecfecc/.tmp/b/d884bbd1c7ec4a5684c0f96c48550697 2024-11-12T14:43:28,367 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40559/hbase/data/default/testReplayEditsWrittenIntoWAL/18980687202e69e67ba6536f50ecfecc/.tmp/c/1a52e5baecad4e58a7f875bd8b3e31b1 is 62, key is testReplayEditsWrittenIntoWAL/c:100/1731422603412/Put/seqid=0 2024-11-12T14:43:28,375 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36175 is added to blk_1073741862_1039 (size=20825) 2024-11-12T14:43:28,376 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45975 is added to blk_1073741862_1039 (size=20825) 2024-11-12T14:43:28,376 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37395 is added to blk_1073741862_1039 (size=20825) 2024-11-12T14:43:28,377 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=14.63 KB at sequenceid=2262 (bloomFilter=true), to=hdfs://localhost:40559/hbase/data/default/testReplayEditsWrittenIntoWAL/18980687202e69e67ba6536f50ecfecc/.tmp/c/1a52e5baecad4e58a7f875bd8b3e31b1 2024-11-12T14:43:28,386 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40559/hbase/data/default/testReplayEditsWrittenIntoWAL/18980687202e69e67ba6536f50ecfecc/.tmp/b/d884bbd1c7ec4a5684c0f96c48550697 as hdfs://localhost:40559/hbase/data/default/testReplayEditsWrittenIntoWAL/18980687202e69e67ba6536f50ecfecc/b/d884bbd1c7ec4a5684c0f96c48550697 2024-11-12T14:43:28,396 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40559/hbase/data/default/testReplayEditsWrittenIntoWAL/18980687202e69e67ba6536f50ecfecc/b/d884bbd1c7ec4a5684c0f96c48550697, entries=492, sequenceid=2262, filesize=34.3 K 2024-11-12T14:43:28,398 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40559/hbase/data/default/testReplayEditsWrittenIntoWAL/18980687202e69e67ba6536f50ecfecc/.tmp/c/1a52e5baecad4e58a7f875bd8b3e31b1 as hdfs://localhost:40559/hbase/data/default/testReplayEditsWrittenIntoWAL/18980687202e69e67ba6536f50ecfecc/c/1a52e5baecad4e58a7f875bd8b3e31b1 2024-11-12T14:43:28,407 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40559/hbase/data/default/testReplayEditsWrittenIntoWAL/18980687202e69e67ba6536f50ecfecc/c/1a52e5baecad4e58a7f875bd8b3e31b1, entries=262, sequenceid=2262, filesize=20.3 K 2024-11-12T14:43:28,408 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~42.49 KB/43512, heapSize ~99.83 KB/102224, currentSize=0 B/0 for 18980687202e69e67ba6536f50ecfecc in 100ms, sequenceid=2262, compaction requested=false; wal=null 2024-11-12T14:43:28,423 WARN [Time-limited test {}] regionserver.HRegion(5722): No family for cell testReplayEditsWrittenIntoWAL/another family:testReplayEditsWrittenIntoWAL/1731422603464/Put/vlen=29/seqid=0 in region testReplayEditsWrittenIntoWAL,,1731422603141.18980687202e69e67ba6536f50ecfecc. 2024-11-12T14:43:28,428 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 3001, skipped 1, firstSequenceIdInLog=1, maxSequenceIdInLog=3002, path=hdfs://localhost:40559/hbase/data/default/testReplayEditsWrittenIntoWAL/18980687202e69e67ba6536f50ecfecc/recovered.edits/0000000000000003002 2024-11-12T14:43:28,429 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-11-12T14:43:28,429 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 18980687202e69e67ba6536f50ecfecc 3/3 column families, dataSize=41.85 KB heapSize=98.89 KB 2024-11-12T14:43:28,441 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40559/hbase/data/default/testReplayEditsWrittenIntoWAL/18980687202e69e67ba6536f50ecfecc/.tmp/c/4a813562484b411c9733c3a577cee6bd is 62, key is testReplayEditsWrittenIntoWAL/c:262/1731422603422/Put/seqid=0 2024-11-12T14:43:28,469 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45975 is added to blk_1073741863_1040 (size=50301) 2024-11-12T14:43:28,471 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36175 is added to blk_1073741863_1040 (size=50301) 2024-11-12T14:43:28,472 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37395 is added to blk_1073741863_1040 (size=50301) 2024-11-12T14:43:28,474 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=41.85 KB at sequenceid=3002 (bloomFilter=true), to=hdfs://localhost:40559/hbase/data/default/testReplayEditsWrittenIntoWAL/18980687202e69e67ba6536f50ecfecc/.tmp/c/4a813562484b411c9733c3a577cee6bd 2024-11-12T14:43:28,485 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 4a813562484b411c9733c3a577cee6bd 2024-11-12T14:43:28,487 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40559/hbase/data/default/testReplayEditsWrittenIntoWAL/18980687202e69e67ba6536f50ecfecc/.tmp/c/4a813562484b411c9733c3a577cee6bd as hdfs://localhost:40559/hbase/data/default/testReplayEditsWrittenIntoWAL/18980687202e69e67ba6536f50ecfecc/c/4a813562484b411c9733c3a577cee6bd 2024-11-12T14:43:28,495 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 4a813562484b411c9733c3a577cee6bd 2024-11-12T14:43:28,496 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40559/hbase/data/default/testReplayEditsWrittenIntoWAL/18980687202e69e67ba6536f50ecfecc/c/4a813562484b411c9733c3a577cee6bd, entries=739, sequenceid=3002, filesize=49.1 K 2024-11-12T14:43:28,496 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~41.85 KB/42854, heapSize ~98.38 KB/100736, currentSize=0 B/0 for 18980687202e69e67ba6536f50ecfecc in 67ms, sequenceid=3002, compaction requested=false; wal=null 2024-11-12T14:43:28,497 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:40559/hbase/data/default/testReplayEditsWrittenIntoWAL/18980687202e69e67ba6536f50ecfecc/recovered.edits/0000000000000003002 2024-11-12T14:43:28,498 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 18980687202e69e67ba6536f50ecfecc 2024-11-12T14:43:28,498 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 18980687202e69e67ba6536f50ecfecc 2024-11-12T14:43:28,499 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenIntoWAL descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-12T14:43:28,501 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 18980687202e69e67ba6536f50ecfecc 2024-11-12T14:43:28,503 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40559/hbase/data/default/testReplayEditsWrittenIntoWAL/18980687202e69e67ba6536f50ecfecc/recovered.edits/3002.seqid, newMaxSeqId=3002, maxSeqId=1 2024-11-12T14:43:28,504 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 18980687202e69e67ba6536f50ecfecc; next sequenceid=3003; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=204800, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68685732, jitterRate=0.023497164249420166}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-12T14:43:28,505 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 18980687202e69e67ba6536f50ecfecc: Writing region info on filesystem at 1731422607699Initializing all the Stores at 1731422607701 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731422607701Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731422607701Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731422607701Cleaning up temporary data from old regions at 1731422608498 (+797 ms)Region opened successfully at 1731422608505 (+7 ms) 2024-11-12T14:43:28,580 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 18980687202e69e67ba6536f50ecfecc, disabling compactions & flushes 2024-11-12T14:43:28,580 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1731422603141.18980687202e69e67ba6536f50ecfecc. 2024-11-12T14:43:28,580 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1731422603141.18980687202e69e67ba6536f50ecfecc. 2024-11-12T14:43:28,580 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1731422603141.18980687202e69e67ba6536f50ecfecc. after waiting 0 ms 2024-11-12T14:43:28,580 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1731422603141.18980687202e69e67ba6536f50ecfecc. 2024-11-12T14:43:28,583 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1731422603141.18980687202e69e67ba6536f50ecfecc. 2024-11-12T14:43:28,583 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 18980687202e69e67ba6536f50ecfecc: Waiting for close lock at 1731422608580Disabling compacts and flushes for region at 1731422608580Disabling writes for close at 1731422608580Writing region close event to WAL at 1731422608583 (+3 ms)Closed at 1731422608583 2024-11-12T14:43:28,587 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45975 is added to blk_1073741857_1034 (size=175) 2024-11-12T14:43:28,587 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36175 is added to blk_1073741857_1034 (size=175) 2024-11-12T14:43:28,587 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37395 is added to blk_1073741857_1034 (size=175) 2024-11-12T14:43:28,591 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-12T14:43:28,591 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1731422607653) 2024-11-12T14:43:28,609 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsWrittenIntoWAL Thread=406 (was 390) Potentially hanging thread: IPC Client (1685805399) connection to localhost/127.0.0.1:40559 from jenkins.replay.wal.secondtime java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:35357 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-2113513671_22 at /127.0.0.1:47348 [Waiting for operation #12] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-12-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5a5e6c5c-3b82-c3b3-1527-3525a941b154/cluster_22b4e46d-f388-f2af-b32c-f9fd886af812/data/data6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkins.replay.wal.secondtime@localhost:40559 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5a5e6c5c-3b82-c3b3-1527-3525a941b154/cluster_22b4e46d-f388-f2af-b32c-f9fd886af812/data/data5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-12-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1685805399) connection to localhost/127.0.0.1:35357 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-2113513671_22 at /127.0.0.1:46268 [Waiting for operation #22] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1685805399) connection to localhost/127.0.0.1:41811 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:41811 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5a5e6c5c-3b82-c3b3-1527-3525a941b154/cluster_22b4e46d-f388-f2af-b32c-f9fd886af812/data/data4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-2113513671_22 at /127.0.0.1:47310 [Waiting for operation #6] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-12-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5a5e6c5c-3b82-c3b3-1527-3525a941b154/cluster_22b4e46d-f388-f2af-b32c-f9fd886af812/data/data3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-2113513671_22 at /127.0.0.1:50608 [Waiting for operation #10] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=865 (was 785) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=298 (was 325), ProcessCount=11 (was 11), AvailableMemoryMB=7650 (was 7785) 2024-11-12T14:43:28,621 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#test2727 Thread=406, OpenFileDescriptor=865, MaxFileDescriptor=1048576, SystemLoadAverage=298, ProcessCount=11, AvailableMemoryMB=7649 2024-11-12T14:43:28,635 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-12T14:43:28,638 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-12T14:43:28,639 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-12T14:43:28,642 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-67373600, suffix=, logDir=hdfs://localhost:40559/hbase/WALs/hregion-67373600, archiveDir=hdfs://localhost:40559/hbase/oldWALs, maxLogs=32 2024-11-12T14:43:28,656 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-67373600/hregion-67373600.1731422608643, exclude list is [], retry=0 2024-11-12T14:43:28,659 DEBUG [AsyncFSWAL-14-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36175,DS-31e7ba0a-e37c-4fe6-8d2a-a4846cefa408,DISK] 2024-11-12T14:43:28,659 DEBUG [AsyncFSWAL-14-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45975,DS-80b047b6-34d2-41c0-99c3-5cd61ca40075,DISK] 2024-11-12T14:43:28,659 DEBUG [AsyncFSWAL-14-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37395,DS-7317e4d9-cef6-4b85-a947-106c3c917da9,DISK] 2024-11-12T14:43:28,686 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-67373600/hregion-67373600.1731422608643 2024-11-12T14:43:28,686 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36027:36027),(127.0.0.1/127.0.0.1:43643:43643),(127.0.0.1/127.0.0.1:40227:40227)] 2024-11-12T14:43:28,686 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 60b147315217af26c8b0a450e08346c0, NAME => 'test2727,,1731422608636.60b147315217af26c8b0a450e08346c0.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='test2727', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40559/hbase 2024-11-12T14:43:28,697 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36175 is added to blk_1073741865_1042 (size=43) 2024-11-12T14:43:28,697 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45975 is added to blk_1073741865_1042 (size=43) 2024-11-12T14:43:28,697 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37395 is added to blk_1073741865_1042 (size=43) 2024-11-12T14:43:28,698 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated test2727,,1731422608636.60b147315217af26c8b0a450e08346c0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T14:43:28,700 INFO [StoreOpener-60b147315217af26c8b0a450e08346c0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 60b147315217af26c8b0a450e08346c0 2024-11-12T14:43:28,702 INFO [StoreOpener-60b147315217af26c8b0a450e08346c0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 60b147315217af26c8b0a450e08346c0 columnFamilyName a 2024-11-12T14:43:28,702 DEBUG [StoreOpener-60b147315217af26c8b0a450e08346c0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:43:28,703 INFO [StoreOpener-60b147315217af26c8b0a450e08346c0-1 {}] regionserver.HStore(327): Store=60b147315217af26c8b0a450e08346c0/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T14:43:28,703 INFO [StoreOpener-60b147315217af26c8b0a450e08346c0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 60b147315217af26c8b0a450e08346c0 2024-11-12T14:43:28,705 INFO [StoreOpener-60b147315217af26c8b0a450e08346c0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 60b147315217af26c8b0a450e08346c0 columnFamilyName b 2024-11-12T14:43:28,705 DEBUG [StoreOpener-60b147315217af26c8b0a450e08346c0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:43:28,706 INFO [StoreOpener-60b147315217af26c8b0a450e08346c0-1 {}] regionserver.HStore(327): Store=60b147315217af26c8b0a450e08346c0/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T14:43:28,706 INFO [StoreOpener-60b147315217af26c8b0a450e08346c0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 60b147315217af26c8b0a450e08346c0 2024-11-12T14:43:28,707 INFO [StoreOpener-60b147315217af26c8b0a450e08346c0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 60b147315217af26c8b0a450e08346c0 columnFamilyName c 2024-11-12T14:43:28,707 DEBUG [StoreOpener-60b147315217af26c8b0a450e08346c0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:43:28,708 INFO [StoreOpener-60b147315217af26c8b0a450e08346c0-1 {}] regionserver.HStore(327): Store=60b147315217af26c8b0a450e08346c0/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T14:43:28,708 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 60b147315217af26c8b0a450e08346c0 2024-11-12T14:43:28,709 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40559/hbase/data/default/test2727/60b147315217af26c8b0a450e08346c0 2024-11-12T14:43:28,709 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40559/hbase/data/default/test2727/60b147315217af26c8b0a450e08346c0 2024-11-12T14:43:28,711 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 60b147315217af26c8b0a450e08346c0 2024-11-12T14:43:28,711 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 60b147315217af26c8b0a450e08346c0 2024-11-12T14:43:28,711 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table test2727 descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-12T14:43:28,713 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 60b147315217af26c8b0a450e08346c0 2024-11-12T14:43:28,717 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40559/hbase/data/default/test2727/60b147315217af26c8b0a450e08346c0/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-12T14:43:28,717 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 60b147315217af26c8b0a450e08346c0; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71566986, jitterRate=0.0664311945438385}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-12T14:43:28,718 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 60b147315217af26c8b0a450e08346c0: Writing region info on filesystem at 1731422608699Initializing all the Stores at 1731422608700 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731422608700Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731422608700Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731422608700Cleaning up temporary data from old regions at 1731422608711 (+11 ms)Region opened successfully at 1731422608718 (+7 ms) 2024-11-12T14:43:28,718 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 60b147315217af26c8b0a450e08346c0, disabling compactions & flushes 2024-11-12T14:43:28,718 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region test2727,,1731422608636.60b147315217af26c8b0a450e08346c0. 2024-11-12T14:43:28,719 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on test2727,,1731422608636.60b147315217af26c8b0a450e08346c0. 2024-11-12T14:43:28,719 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on test2727,,1731422608636.60b147315217af26c8b0a450e08346c0. after waiting 0 ms 2024-11-12T14:43:28,719 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region test2727,,1731422608636.60b147315217af26c8b0a450e08346c0. 2024-11-12T14:43:28,719 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed test2727,,1731422608636.60b147315217af26c8b0a450e08346c0. 2024-11-12T14:43:28,719 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 60b147315217af26c8b0a450e08346c0: Waiting for close lock at 1731422608718Disabling compacts and flushes for region at 1731422608718Disabling writes for close at 1731422608719 (+1 ms)Writing region close event to WAL at 1731422608719Closed at 1731422608719 2024-11-12T14:43:28,722 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37395 is added to blk_1073741864_1041 (size=175) 2024-11-12T14:43:28,722 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36175 is added to blk_1073741864_1041 (size=175) 2024-11-12T14:43:28,723 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45975 is added to blk_1073741864_1041 (size=175) 2024-11-12T14:43:28,727 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-12T14:43:28,728 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-67373600:(num 1731422608643) 2024-11-12T14:43:28,728 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-12T14:43:28,730 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:40559/hbase/WALs/test2727-manual,16010,1731422608634, archiveDir=hdfs://localhost:40559/hbase/oldWALs, maxLogs=32 2024-11-12T14:43:28,745 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/test2727-manual,16010,1731422608634/wal.1731422608731, exclude list is [], retry=0 2024-11-12T14:43:28,749 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45975,DS-80b047b6-34d2-41c0-99c3-5cd61ca40075,DISK] 2024-11-12T14:43:28,749 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37395,DS-7317e4d9-cef6-4b85-a947-106c3c917da9,DISK] 2024-11-12T14:43:28,750 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36175,DS-31e7ba0a-e37c-4fe6-8d2a-a4846cefa408,DISK] 2024-11-12T14:43:28,786 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/test2727-manual,16010,1731422608634/wal.1731422608731 2024-11-12T14:43:28,786 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43643:43643),(127.0.0.1/127.0.0.1:40227:40227),(127.0.0.1/127.0.0.1:36027:36027)] 2024-11-12T14:43:28,975 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37395 is added to blk_1073741866_1043 (size=371388) 2024-11-12T14:43:28,975 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45975 is added to blk_1073741866_1043 (size=371388) 2024-11-12T14:43:28,975 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /hbase/WALs/test2727-manual,16010,1731422608634/wal.1731422608731 not finished, retry = 0 2024-11-12T14:43:28,975 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36175 is added to blk_1073741866_1043 (size=371388) 2024-11-12T14:43:29,097 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:40559/hbase/WALs/test2727-manual,16010,1731422608634/wal.1731422608731, size=362.7 K (371388bytes) 2024-11-12T14:43:29,097 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:40559/hbase/WALs/test2727-manual,16010,1731422608634/wal.1731422608731 2024-11-12T14:43:29,097 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:40559/hbase/WALs/test2727-manual,16010,1731422608634/wal.1731422608731 after 0ms 2024-11-12T14:43:29,125 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-12T14:43:29,127 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:40559/hbase/WALs/test2727-manual,16010,1731422608634/wal.1731422608731 took 31ms 2024-11-12T14:43:29,131 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal.1731422608731.temp 2024-11-12T14:43:29,167 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:40559/hbase/data/default/test2727/60b147315217af26c8b0a450e08346c0/recovered.edits/0000000000000000001-wal.1731422608731.temp 2024-11-12T14:43:29,176 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:40559/hbase/WALs/test2727-manual,16010,1731422608634/wal.1731422608731 so closing down 2024-11-12T14:43:29,176 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-12T14:43:29,189 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-12T14:43:29,194 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45975 is added to blk_1073741867_1044 (size=371388) 2024-11-12T14:43:29,194 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36175 is added to blk_1073741867_1044 (size=371388) 2024-11-12T14:43:29,194 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37395 is added to blk_1073741867_1044 (size=371388) 2024-11-12T14:43:29,195 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:40559/hbase/data/default/test2727/60b147315217af26c8b0a450e08346c0/recovered.edits/0000000000000000001-wal.1731422608731.temp (wrote 3000 edits, skipped 0 edits in 20 ms) 2024-11-12T14:43:29,197 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:40559/hbase/data/default/test2727/60b147315217af26c8b0a450e08346c0/recovered.edits/0000000000000000001-wal.1731422608731.temp to hdfs://localhost:40559/hbase/data/default/test2727/60b147315217af26c8b0a450e08346c0/recovered.edits/0000000000000003000 2024-11-12T14:43:29,197 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 3000 edits across 1 Regions in 70 ms; skipped=0; WAL=hdfs://localhost:40559/hbase/WALs/test2727-manual,16010,1731422608634/wal.1731422608731, size=362.7 K, length=371388, corrupted=false, cancelled=false 2024-11-12T14:43:29,197 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:40559/hbase/WALs/test2727-manual,16010,1731422608634/wal.1731422608731, journal: Splitting hdfs://localhost:40559/hbase/WALs/test2727-manual,16010,1731422608634/wal.1731422608731, size=362.7 K (371388bytes) at 1731422609097Split 1024 edits, skipped 0 edits. at 1731422609146 (+49 ms)Split 2048 edits, skipped 0 edits. at 1731422609162 (+16 ms)Creating recovered edits writer path=hdfs://localhost:40559/hbase/data/default/test2727/60b147315217af26c8b0a450e08346c0/recovered.edits/0000000000000000001-wal.1731422608731.temp at 1731422609167 (+5 ms)Finishing writing output for hdfs://localhost:40559/hbase/WALs/test2727-manual,16010,1731422608634/wal.1731422608731 so closing down at 1731422609176 (+9 ms)3 split writer threads finished at 1731422609189 (+13 ms)Closed recovered edits writer path=hdfs://localhost:40559/hbase/data/default/test2727/60b147315217af26c8b0a450e08346c0/recovered.edits/0000000000000000001-wal.1731422608731.temp (wrote 3000 edits, skipped 0 edits in 20 ms) at 1731422609195 (+6 ms)Rename recovered edits hdfs://localhost:40559/hbase/data/default/test2727/60b147315217af26c8b0a450e08346c0/recovered.edits/0000000000000000001-wal.1731422608731.temp to hdfs://localhost:40559/hbase/data/default/test2727/60b147315217af26c8b0a450e08346c0/recovered.edits/0000000000000003000 at 1731422609197 (+2 ms)Processed 3000 edits across 1 Regions in 70 ms; skipped=0; WAL=hdfs://localhost:40559/hbase/WALs/test2727-manual,16010,1731422608634/wal.1731422608731, size=362.7 K, length=371388, corrupted=false, cancelled=false at 1731422609197 2024-11-12T14:43:29,199 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:40559/hbase/WALs/test2727-manual,16010,1731422608634/wal.1731422608731 to hdfs://localhost:40559/hbase/oldWALs/wal.1731422608731 2024-11-12T14:43:29,200 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:40559/hbase/data/default/test2727/60b147315217af26c8b0a450e08346c0/recovered.edits/0000000000000003000 2024-11-12T14:43:29,200 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-12T14:43:29,202 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:40559/hbase/WALs/test2727-manual,16010,1731422608634, archiveDir=hdfs://localhost:40559/hbase/oldWALs, maxLogs=32 2024-11-12T14:43:29,217 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/test2727-manual,16010,1731422608634/wal.1731422609203, exclude list is [], retry=0 2024-11-12T14:43:29,220 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37395,DS-7317e4d9-cef6-4b85-a947-106c3c917da9,DISK] 2024-11-12T14:43:29,220 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36175,DS-31e7ba0a-e37c-4fe6-8d2a-a4846cefa408,DISK] 2024-11-12T14:43:29,221 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45975,DS-80b047b6-34d2-41c0-99c3-5cd61ca40075,DISK] 2024-11-12T14:43:29,252 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/test2727-manual,16010,1731422608634/wal.1731422609203 2024-11-12T14:43:29,252 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40227:40227),(127.0.0.1/127.0.0.1:36027:36027),(127.0.0.1/127.0.0.1:43643:43643)] 2024-11-12T14:43:29,366 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-12T14:43:29,366 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-12T14:43:29,368 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-12T14:43:29,368 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-12T14:43:29,369 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-12T14:43:29,369 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-12T14:43:29,371 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenIntoWAL 2024-11-12T14:43:29,371 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenIntoWAL Metrics about Tables on a single HBase RegionServer 2024-11-12T14:43:29,399 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37395 is added to blk_1073741868_1045 (size=371515) 2024-11-12T14:43:29,399 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45975 is added to blk_1073741868_1045 (size=371515) 2024-11-12T14:43:29,400 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36175 is added to blk_1073741868_1045 (size=371515) 2024-11-12T14:43:29,416 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:40559/hbase/WALs/test2727-manual,16010,1731422608634/wal.1731422609203, size=362.8 K (371515bytes) 2024-11-12T14:43:29,416 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:40559/hbase/WALs/test2727-manual,16010,1731422608634/wal.1731422609203 2024-11-12T14:43:29,417 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:40559/hbase/WALs/test2727-manual,16010,1731422608634/wal.1731422609203 after 1ms 2024-11-12T14:43:29,445 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-12T14:43:29,446 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:40559/hbase/WALs/test2727-manual,16010,1731422608634/wal.1731422609203 took 30ms 2024-11-12T14:43:29,450 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000003001-wal.1731422609203.temp 2024-11-12T14:43:29,496 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:40559/hbase/WALs/test2727-manual,16010,1731422608634/wal.1731422609203 so closing down 2024-11-12T14:43:29,496 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-12T14:43:29,515 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:40559/hbase/data/default/test2727/60b147315217af26c8b0a450e08346c0/recovered.edits/0000000000000003001-wal.1731422609203.temp 2024-11-12T14:43:29,531 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-12T14:43:29,540 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37395 is added to blk_1073741869_1046 (size=371515) 2024-11-12T14:43:29,541 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45975 is added to blk_1073741869_1046 (size=371515) 2024-11-12T14:43:29,541 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:40559/hbase/data/default/test2727/60b147315217af26c8b0a450e08346c0/recovered.edits/0000000000000003001-wal.1731422609203.temp (wrote 3000 edits, skipped 0 edits in 16 ms) 2024-11-12T14:43:29,541 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36175 is added to blk_1073741869_1046 (size=371515) 2024-11-12T14:43:29,542 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:40559/hbase/data/default/test2727/60b147315217af26c8b0a450e08346c0/recovered.edits/0000000000000003001-wal.1731422609203.temp to hdfs://localhost:40559/hbase/data/default/test2727/60b147315217af26c8b0a450e08346c0/recovered.edits/0000000000000006000 2024-11-12T14:43:29,543 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 3000 edits across 1 Regions in 97 ms; skipped=0; WAL=hdfs://localhost:40559/hbase/WALs/test2727-manual,16010,1731422608634/wal.1731422609203, size=362.8 K, length=371515, corrupted=false, cancelled=false 2024-11-12T14:43:29,543 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:40559/hbase/WALs/test2727-manual,16010,1731422608634/wal.1731422609203, journal: Splitting hdfs://localhost:40559/hbase/WALs/test2727-manual,16010,1731422608634/wal.1731422609203, size=362.8 K (371515bytes) at 1731422609416Split 1024 edits, skipped 0 edits. at 1731422609459 (+43 ms)Split 2048 edits, skipped 0 edits. at 1731422609487 (+28 ms)Finishing writing output for hdfs://localhost:40559/hbase/WALs/test2727-manual,16010,1731422608634/wal.1731422609203 so closing down at 1731422609496 (+9 ms)Creating recovered edits writer path=hdfs://localhost:40559/hbase/data/default/test2727/60b147315217af26c8b0a450e08346c0/recovered.edits/0000000000000003001-wal.1731422609203.temp at 1731422609515 (+19 ms)3 split writer threads finished at 1731422609531 (+16 ms)Closed recovered edits writer path=hdfs://localhost:40559/hbase/data/default/test2727/60b147315217af26c8b0a450e08346c0/recovered.edits/0000000000000003001-wal.1731422609203.temp (wrote 3000 edits, skipped 0 edits in 16 ms) at 1731422609541 (+10 ms)Rename recovered edits hdfs://localhost:40559/hbase/data/default/test2727/60b147315217af26c8b0a450e08346c0/recovered.edits/0000000000000003001-wal.1731422609203.temp to hdfs://localhost:40559/hbase/data/default/test2727/60b147315217af26c8b0a450e08346c0/recovered.edits/0000000000000006000 at 1731422609543 (+2 ms)Processed 3000 edits across 1 Regions in 97 ms; skipped=0; WAL=hdfs://localhost:40559/hbase/WALs/test2727-manual,16010,1731422608634/wal.1731422609203, size=362.8 K, length=371515, corrupted=false, cancelled=false at 1731422609543 2024-11-12T14:43:29,545 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:40559/hbase/WALs/test2727-manual,16010,1731422608634/wal.1731422609203 to hdfs://localhost:40559/hbase/oldWALs/wal.1731422609203 2024-11-12T14:43:29,546 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:40559/hbase/data/default/test2727/60b147315217af26c8b0a450e08346c0/recovered.edits/0000000000000006000 2024-11-12T14:43:29,546 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-12T14:43:29,548 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:40559/hbase/WALs/test2727-manual,16010,1731422608634, archiveDir=hdfs://localhost:40559/hbase/oldWALs, maxLogs=32 2024-11-12T14:43:29,562 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/test2727-manual,16010,1731422608634/wal.1731422609549, exclude list is [], retry=0 2024-11-12T14:43:29,566 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37395,DS-7317e4d9-cef6-4b85-a947-106c3c917da9,DISK] 2024-11-12T14:43:29,566 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36175,DS-31e7ba0a-e37c-4fe6-8d2a-a4846cefa408,DISK] 2024-11-12T14:43:29,566 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45975,DS-80b047b6-34d2-41c0-99c3-5cd61ca40075,DISK] 2024-11-12T14:43:29,594 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/test2727-manual,16010,1731422608634/wal.1731422609549 2024-11-12T14:43:29,594 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40227:40227),(127.0.0.1/127.0.0.1:36027:36027),(127.0.0.1/127.0.0.1:43643:43643)] 2024-11-12T14:43:29,594 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 60b147315217af26c8b0a450e08346c0, NAME => 'test2727,,1731422608636.60b147315217af26c8b0a450e08346c0.', STARTKEY => '', ENDKEY => ''} 2024-11-12T14:43:29,594 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated test2727,,1731422608636.60b147315217af26c8b0a450e08346c0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T14:43:29,595 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 60b147315217af26c8b0a450e08346c0 2024-11-12T14:43:29,595 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 60b147315217af26c8b0a450e08346c0 2024-11-12T14:43:29,596 INFO [StoreOpener-60b147315217af26c8b0a450e08346c0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 60b147315217af26c8b0a450e08346c0 2024-11-12T14:43:29,597 INFO [StoreOpener-60b147315217af26c8b0a450e08346c0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 60b147315217af26c8b0a450e08346c0 columnFamilyName a 2024-11-12T14:43:29,597 DEBUG [StoreOpener-60b147315217af26c8b0a450e08346c0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:43:29,598 INFO [StoreOpener-60b147315217af26c8b0a450e08346c0-1 {}] regionserver.HStore(327): Store=60b147315217af26c8b0a450e08346c0/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T14:43:29,598 INFO [StoreOpener-60b147315217af26c8b0a450e08346c0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 60b147315217af26c8b0a450e08346c0 2024-11-12T14:43:29,599 INFO [StoreOpener-60b147315217af26c8b0a450e08346c0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 60b147315217af26c8b0a450e08346c0 columnFamilyName b 2024-11-12T14:43:29,600 DEBUG [StoreOpener-60b147315217af26c8b0a450e08346c0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:43:29,600 INFO [StoreOpener-60b147315217af26c8b0a450e08346c0-1 {}] regionserver.HStore(327): Store=60b147315217af26c8b0a450e08346c0/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T14:43:29,600 INFO [StoreOpener-60b147315217af26c8b0a450e08346c0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 60b147315217af26c8b0a450e08346c0 2024-11-12T14:43:29,602 INFO [StoreOpener-60b147315217af26c8b0a450e08346c0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 60b147315217af26c8b0a450e08346c0 columnFamilyName c 2024-11-12T14:43:29,602 DEBUG [StoreOpener-60b147315217af26c8b0a450e08346c0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:43:29,602 INFO [StoreOpener-60b147315217af26c8b0a450e08346c0-1 {}] regionserver.HStore(327): Store=60b147315217af26c8b0a450e08346c0/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T14:43:29,603 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 60b147315217af26c8b0a450e08346c0 2024-11-12T14:43:29,603 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40559/hbase/data/default/test2727/60b147315217af26c8b0a450e08346c0 2024-11-12T14:43:29,606 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 2 recovered edits file(s) under hdfs://localhost:40559/hbase/data/default/test2727/60b147315217af26c8b0a450e08346c0 2024-11-12T14:43:29,607 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:40559/hbase/data/default/test2727/60b147315217af26c8b0a450e08346c0/recovered.edits/0000000000000003000 2024-11-12T14:43:29,633 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-12T14:43:29,670 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 3000, skipped 0, firstSequenceIdInLog=1, maxSequenceIdInLog=3000, path=hdfs://localhost:40559/hbase/data/default/test2727/60b147315217af26c8b0a450e08346c0/recovered.edits/0000000000000003000 2024-11-12T14:43:29,672 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:40559/hbase/data/default/test2727/60b147315217af26c8b0a450e08346c0/recovered.edits/0000000000000006000 2024-11-12T14:43:29,700 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-12T14:43:29,736 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 3000, skipped 0, firstSequenceIdInLog=3001, maxSequenceIdInLog=6000, path=hdfs://localhost:40559/hbase/data/default/test2727/60b147315217af26c8b0a450e08346c0/recovered.edits/0000000000000006000 2024-11-12T14:43:29,736 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 60b147315217af26c8b0a450e08346c0 3/3 column families, dataSize=215.51 KB heapSize=657 KB 2024-11-12T14:43:29,758 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40559/hbase/data/default/test2727/60b147315217af26c8b0a450e08346c0/.tmp/a/eebe7af0c4274bf0b8a33b6e7194e0df is 41, key is test2727/a:100/1731422609258/Put/seqid=0 2024-11-12T14:43:29,764 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45975 is added to blk_1073741871_1048 (size=84227) 2024-11-12T14:43:29,764 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37395 is added to blk_1073741871_1048 (size=84227) 2024-11-12T14:43:29,764 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36175 is added to blk_1073741871_1048 (size=84227) 2024-11-12T14:43:29,765 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=71.84 KB at sequenceid=6000 (bloomFilter=true), to=hdfs://localhost:40559/hbase/data/default/test2727/60b147315217af26c8b0a450e08346c0/.tmp/a/eebe7af0c4274bf0b8a33b6e7194e0df 2024-11-12T14:43:29,798 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40559/hbase/data/default/test2727/60b147315217af26c8b0a450e08346c0/.tmp/b/b30877b271b24773807f923c67f3be89 is 41, key is test2727/b:100/1731422609307/Put/seqid=0 2024-11-12T14:43:29,802 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37395 is added to blk_1073741872_1049 (size=84609) 2024-11-12T14:43:29,802 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36175 is added to blk_1073741872_1049 (size=84609) 2024-11-12T14:43:29,803 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45975 is added to blk_1073741872_1049 (size=84609) 2024-11-12T14:43:29,804 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=71.84 KB at sequenceid=6000 (bloomFilter=true), to=hdfs://localhost:40559/hbase/data/default/test2727/60b147315217af26c8b0a450e08346c0/.tmp/b/b30877b271b24773807f923c67f3be89 2024-11-12T14:43:29,829 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40559/hbase/data/default/test2727/60b147315217af26c8b0a450e08346c0/.tmp/c/b51d8b948054495dbb47b8d285644765 is 41, key is test2727/c:100/1731422609350/Put/seqid=0 2024-11-12T14:43:29,835 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45975 is added to blk_1073741873_1050 (size=84609) 2024-11-12T14:43:29,835 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37395 is added to blk_1073741873_1050 (size=84609) 2024-11-12T14:43:29,835 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36175 is added to blk_1073741873_1050 (size=84609) 2024-11-12T14:43:29,835 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=71.84 KB at sequenceid=6000 (bloomFilter=true), to=hdfs://localhost:40559/hbase/data/default/test2727/60b147315217af26c8b0a450e08346c0/.tmp/c/b51d8b948054495dbb47b8d285644765 2024-11-12T14:43:29,842 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40559/hbase/data/default/test2727/60b147315217af26c8b0a450e08346c0/.tmp/a/eebe7af0c4274bf0b8a33b6e7194e0df as hdfs://localhost:40559/hbase/data/default/test2727/60b147315217af26c8b0a450e08346c0/a/eebe7af0c4274bf0b8a33b6e7194e0df 2024-11-12T14:43:29,850 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40559/hbase/data/default/test2727/60b147315217af26c8b0a450e08346c0/a/eebe7af0c4274bf0b8a33b6e7194e0df, entries=2000, sequenceid=6000, filesize=82.3 K 2024-11-12T14:43:29,851 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40559/hbase/data/default/test2727/60b147315217af26c8b0a450e08346c0/.tmp/b/b30877b271b24773807f923c67f3be89 as hdfs://localhost:40559/hbase/data/default/test2727/60b147315217af26c8b0a450e08346c0/b/b30877b271b24773807f923c67f3be89 2024-11-12T14:43:29,859 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40559/hbase/data/default/test2727/60b147315217af26c8b0a450e08346c0/b/b30877b271b24773807f923c67f3be89, entries=2000, sequenceid=6000, filesize=82.6 K 2024-11-12T14:43:29,860 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40559/hbase/data/default/test2727/60b147315217af26c8b0a450e08346c0/.tmp/c/b51d8b948054495dbb47b8d285644765 as hdfs://localhost:40559/hbase/data/default/test2727/60b147315217af26c8b0a450e08346c0/c/b51d8b948054495dbb47b8d285644765 2024-11-12T14:43:29,870 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40559/hbase/data/default/test2727/60b147315217af26c8b0a450e08346c0/c/b51d8b948054495dbb47b8d285644765, entries=2000, sequenceid=6000, filesize=82.6 K 2024-11-12T14:43:29,870 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~215.51 KB/220680, heapSize ~656.95 KB/672720, currentSize=0 B/0 for 60b147315217af26c8b0a450e08346c0 in 134ms, sequenceid=6000, compaction requested=false; wal=null 2024-11-12T14:43:29,871 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:40559/hbase/data/default/test2727/60b147315217af26c8b0a450e08346c0/recovered.edits/0000000000000003000 2024-11-12T14:43:29,872 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:40559/hbase/data/default/test2727/60b147315217af26c8b0a450e08346c0/recovered.edits/0000000000000006000 2024-11-12T14:43:29,873 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 60b147315217af26c8b0a450e08346c0 2024-11-12T14:43:29,873 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 60b147315217af26c8b0a450e08346c0 2024-11-12T14:43:29,874 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table test2727 descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-12T14:43:29,876 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 60b147315217af26c8b0a450e08346c0 2024-11-12T14:43:29,879 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40559/hbase/data/default/test2727/60b147315217af26c8b0a450e08346c0/recovered.edits/6000.seqid, newMaxSeqId=6000, maxSeqId=1 2024-11-12T14:43:29,880 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 60b147315217af26c8b0a450e08346c0; next sequenceid=6001; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65525138, jitterRate=-0.023599356412887573}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-12T14:43:29,882 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 60b147315217af26c8b0a450e08346c0: Writing region info on filesystem at 1731422609595Initializing all the Stores at 1731422609596 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731422609596Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731422609596Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731422609596Obtaining lock to block concurrent updates at 1731422609736 (+140 ms)Preparing flush snapshotting stores in 60b147315217af26c8b0a450e08346c0 at 1731422609736Finished memstore snapshotting test2727,,1731422608636.60b147315217af26c8b0a450e08346c0., syncing WAL and waiting on mvcc, flushsize=dataSize=220680, getHeapSize=672720, getOffHeapSize=0, getCellsCount=6000 at 1731422609736Flushing stores of test2727,,1731422608636.60b147315217af26c8b0a450e08346c0. at 1731422609736Flushing 60b147315217af26c8b0a450e08346c0/a: creating writer at 1731422609737 (+1 ms)Flushing 60b147315217af26c8b0a450e08346c0/a: appending metadata at 1731422609757 (+20 ms)Flushing 60b147315217af26c8b0a450e08346c0/a: closing flushed file at 1731422609757Flushing 60b147315217af26c8b0a450e08346c0/b: creating writer at 1731422609772 (+15 ms)Flushing 60b147315217af26c8b0a450e08346c0/b: appending metadata at 1731422609796 (+24 ms)Flushing 60b147315217af26c8b0a450e08346c0/b: closing flushed file at 1731422609796Flushing 60b147315217af26c8b0a450e08346c0/c: creating writer at 1731422609810 (+14 ms)Flushing 60b147315217af26c8b0a450e08346c0/c: appending metadata at 1731422609829 (+19 ms)Flushing 60b147315217af26c8b0a450e08346c0/c: closing flushed file at 1731422609829Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@638d2f8b: reopening flushed file at 1731422609841 (+12 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@422e87: reopening flushed file at 1731422609850 (+9 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@17191638: reopening flushed file at 1731422609859 (+9 ms)Finished flush of dataSize ~215.51 KB/220680, heapSize ~656.95 KB/672720, currentSize=0 B/0 for 60b147315217af26c8b0a450e08346c0 in 134ms, sequenceid=6000, compaction requested=false; wal=null at 1731422609870 (+11 ms)Cleaning up temporary data from old regions at 1731422609873 (+3 ms)Region opened successfully at 1731422609882 (+9 ms) 2024-11-12T14:43:29,883 DEBUG [Time-limited test {}] wal.AbstractTestWALReplay(320): region.getOpenSeqNum(): 6001, wal3.id: 0 2024-11-12T14:43:29,883 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 60b147315217af26c8b0a450e08346c0, disabling compactions & flushes 2024-11-12T14:43:29,883 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region test2727,,1731422608636.60b147315217af26c8b0a450e08346c0. 2024-11-12T14:43:29,883 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on test2727,,1731422608636.60b147315217af26c8b0a450e08346c0. 2024-11-12T14:43:29,883 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on test2727,,1731422608636.60b147315217af26c8b0a450e08346c0. after waiting 0 ms 2024-11-12T14:43:29,884 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region test2727,,1731422608636.60b147315217af26c8b0a450e08346c0. 2024-11-12T14:43:29,885 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed test2727,,1731422608636.60b147315217af26c8b0a450e08346c0. 2024-11-12T14:43:29,885 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 60b147315217af26c8b0a450e08346c0: Waiting for close lock at 1731422609883Disabling compacts and flushes for region at 1731422609883Disabling writes for close at 1731422609883Writing region close event to WAL at 1731422609885 (+2 ms)Closed at 1731422609885 2024-11-12T14:43:29,888 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36175 is added to blk_1073741870_1047 (size=175) 2024-11-12T14:43:29,888 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37395 is added to blk_1073741870_1047 (size=175) 2024-11-12T14:43:29,888 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45975 is added to blk_1073741870_1047 (size=175) 2024-11-12T14:43:29,891 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-12T14:43:29,891 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1731422609549) 2024-11-12T14:43:29,905 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#test2727 Thread=408 (was 406) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-425049348_22 at /127.0.0.1:44176 [Waiting for operation #13] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-425049348_22 at /127.0.0.1:38556 [Waiting for operation #5] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-14-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-14-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-14-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=927 (was 865) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=298 (was 298), ProcessCount=11 (was 11), AvailableMemoryMB=7420 (was 7649) 2024-11-12T14:43:29,915 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testSequentialEditLogSeqNum Thread=408, OpenFileDescriptor=927, MaxFileDescriptor=1048576, SystemLoadAverage=298, ProcessCount=11, AvailableMemoryMB=7420 2024-11-12T14:43:29,937 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-12T14:43:29,944 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:40559/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1731422609936, archiveDir=hdfs://localhost:40559/hbase/oldWALs, maxLogs=32 2024-11-12T14:43:29,945 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731422609945 2024-11-12T14:43:29,980 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testsequentialeditlogseqnum-manual,16010,1731422609936/wal.1731422609945 2024-11-12T14:43:29,985 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new MockWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43643:43643),(127.0.0.1/127.0.0.1:40227:40227),(127.0.0.1/127.0.0.1:36027:36027)] 2024-11-12T14:43:29,989 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => ed6ff18d38ce2f69abd262db70dc19d1, NAME => 'testSequentialEditLogSeqNum,,1731422609938.ed6ff18d38ce2f69abd262db70dc19d1.', STARTKEY => '', ENDKEY => ''} 2024-11-12T14:43:29,989 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testSequentialEditLogSeqNum,,1731422609938.ed6ff18d38ce2f69abd262db70dc19d1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T14:43:29,989 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for ed6ff18d38ce2f69abd262db70dc19d1 2024-11-12T14:43:29,989 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for ed6ff18d38ce2f69abd262db70dc19d1 2024-11-12T14:43:29,990 WARN [Time-limited test {}] regionserver.HRegionFileSystem(836): hdfs://localhost:40559/hbase/data/default/testSequentialEditLogSeqNum/ed6ff18d38ce2f69abd262db70dc19d1 doesn't exist for region: ed6ff18d38ce2f69abd262db70dc19d1 on table testSequentialEditLogSeqNum 2024-11-12T14:43:29,991 WARN [Time-limited test {}] regionserver.HRegionFileSystem(854): .regioninfo file not found for region: ed6ff18d38ce2f69abd262db70dc19d1 on table testSequentialEditLogSeqNum 2024-11-12T14:43:29,998 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37395 is added to blk_1073741875_1052 (size=62) 2024-11-12T14:43:29,999 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45975 is added to blk_1073741875_1052 (size=62) 2024-11-12T14:43:30,000 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36175 is added to blk_1073741875_1052 (size=62) 2024-11-12T14:43:30,001 INFO [StoreOpener-ed6ff18d38ce2f69abd262db70dc19d1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region ed6ff18d38ce2f69abd262db70dc19d1 2024-11-12T14:43:30,003 INFO [StoreOpener-ed6ff18d38ce2f69abd262db70dc19d1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ed6ff18d38ce2f69abd262db70dc19d1 columnFamilyName a 2024-11-12T14:43:30,003 DEBUG [StoreOpener-ed6ff18d38ce2f69abd262db70dc19d1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:43:30,004 INFO [StoreOpener-ed6ff18d38ce2f69abd262db70dc19d1-1 {}] regionserver.HStore(327): Store=ed6ff18d38ce2f69abd262db70dc19d1/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T14:43:30,004 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for ed6ff18d38ce2f69abd262db70dc19d1 2024-11-12T14:43:30,005 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40559/hbase/data/default/testSequentialEditLogSeqNum/ed6ff18d38ce2f69abd262db70dc19d1 2024-11-12T14:43:30,005 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40559/hbase/data/default/testSequentialEditLogSeqNum/ed6ff18d38ce2f69abd262db70dc19d1 2024-11-12T14:43:30,005 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for ed6ff18d38ce2f69abd262db70dc19d1 2024-11-12T14:43:30,005 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for ed6ff18d38ce2f69abd262db70dc19d1 2024-11-12T14:43:30,007 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for ed6ff18d38ce2f69abd262db70dc19d1 2024-11-12T14:43:30,010 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40559/hbase/data/default/testSequentialEditLogSeqNum/ed6ff18d38ce2f69abd262db70dc19d1/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-12T14:43:30,010 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened ed6ff18d38ce2f69abd262db70dc19d1; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64659387, jitterRate=-0.036500051617622375}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-12T14:43:30,011 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for ed6ff18d38ce2f69abd262db70dc19d1: Writing region info on filesystem at 1731422609989Initializing all the Stores at 1731422610001 (+12 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731422610001Cleaning up temporary data from old regions at 1731422610006 (+5 ms)Region opened successfully at 1731422610011 (+5 ms) 2024-11-12T14:43:30,022 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing ed6ff18d38ce2f69abd262db70dc19d1 1/1 column families, dataSize=770 B heapSize=1.73 KB 2024-11-12T14:43:30,041 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40559/hbase/data/default/testSequentialEditLogSeqNum/ed6ff18d38ce2f69abd262db70dc19d1/.tmp/a/dc36a6713099400b9e928a62830c94c5 is 81, key is testSequentialEditLogSeqNum/a:x0/1731422610011/Put/seqid=0 2024-11-12T14:43:30,047 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37395 is added to blk_1073741876_1053 (size=5833) 2024-11-12T14:43:30,047 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45975 is added to blk_1073741876_1053 (size=5833) 2024-11-12T14:43:30,047 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36175 is added to blk_1073741876_1053 (size=5833) 2024-11-12T14:43:30,048 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=770 B at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:40559/hbase/data/default/testSequentialEditLogSeqNum/ed6ff18d38ce2f69abd262db70dc19d1/.tmp/a/dc36a6713099400b9e928a62830c94c5 2024-11-12T14:43:30,054 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40559/hbase/data/default/testSequentialEditLogSeqNum/ed6ff18d38ce2f69abd262db70dc19d1/.tmp/a/dc36a6713099400b9e928a62830c94c5 as hdfs://localhost:40559/hbase/data/default/testSequentialEditLogSeqNum/ed6ff18d38ce2f69abd262db70dc19d1/a/dc36a6713099400b9e928a62830c94c5 2024-11-12T14:43:30,060 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40559/hbase/data/default/testSequentialEditLogSeqNum/ed6ff18d38ce2f69abd262db70dc19d1/a/dc36a6713099400b9e928a62830c94c5, entries=10, sequenceid=13, filesize=5.7 K 2024-11-12T14:43:30,061 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~770 B/770, heapSize ~1.72 KB/1760, currentSize=0 B/0 for ed6ff18d38ce2f69abd262db70dc19d1 in 39ms, sequenceid=13, compaction requested=false 2024-11-12T14:43:30,061 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for ed6ff18d38ce2f69abd262db70dc19d1: 2024-11-12T14:43:30,069 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T14:43:30,069 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T14:43:30,069 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T14:43:30,069 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T14:43:30,069 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T14:43:30,071 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36175 is added to blk_1073741874_1051 (size=3256) 2024-11-12T14:43:30,072 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37395 is added to blk_1073741874_1051 (size=3256) 2024-11-12T14:43:30,072 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45975 is added to blk_1073741874_1051 (size=3256) 2024-11-12T14:43:30,093 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:40559/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1731422609936/wal.1731422609945, size=3.2 K (3256bytes) 2024-11-12T14:43:30,093 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:40559/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1731422609936/wal.1731422609945 2024-11-12T14:43:30,094 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:40559/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1731422609936/wal.1731422609945 after 1ms 2024-11-12T14:43:30,118 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-12T14:43:30,118 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:40559/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1731422609936/wal.1731422609945 took 25ms 2024-11-12T14:43:30,121 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:40559/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1731422609936/wal.1731422609945 so closing down 2024-11-12T14:43:30,121 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-12T14:43:30,122 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1731422609945.temp 2024-11-12T14:43:30,146 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:40559/hbase/data/default/testSequentialEditLogSeqNum/ed6ff18d38ce2f69abd262db70dc19d1/recovered.edits/0000000000000000003-wal.1731422609945.temp 2024-11-12T14:43:30,147 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-12T14:43:30,153 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45975 is added to blk_1073741877_1054 (size=2650) 2024-11-12T14:43:30,153 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36175 is added to blk_1073741877_1054 (size=2650) 2024-11-12T14:43:30,153 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37395 is added to blk_1073741877_1054 (size=2650) 2024-11-12T14:43:30,154 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:40559/hbase/data/default/testSequentialEditLogSeqNum/ed6ff18d38ce2f69abd262db70dc19d1/recovered.edits/0000000000000000003-wal.1731422609945.temp (wrote 15 edits, skipped 0 edits in 0 ms) 2024-11-12T14:43:30,155 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:40559/hbase/data/default/testSequentialEditLogSeqNum/ed6ff18d38ce2f69abd262db70dc19d1/recovered.edits/0000000000000000003-wal.1731422609945.temp to hdfs://localhost:40559/hbase/data/default/testSequentialEditLogSeqNum/ed6ff18d38ce2f69abd262db70dc19d1/recovered.edits/0000000000000000020 2024-11-12T14:43:30,155 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 17 edits across 1 Regions in 36 ms; skipped=2; WAL=hdfs://localhost:40559/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1731422609936/wal.1731422609945, size=3.2 K, length=3256, corrupted=false, cancelled=false 2024-11-12T14:43:30,155 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:40559/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1731422609936/wal.1731422609945, journal: Splitting hdfs://localhost:40559/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1731422609936/wal.1731422609945, size=3.2 K (3256bytes) at 1731422610093Finishing writing output for hdfs://localhost:40559/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1731422609936/wal.1731422609945 so closing down at 1731422610121 (+28 ms)Creating recovered edits writer path=hdfs://localhost:40559/hbase/data/default/testSequentialEditLogSeqNum/ed6ff18d38ce2f69abd262db70dc19d1/recovered.edits/0000000000000000003-wal.1731422609945.temp at 1731422610146 (+25 ms)3 split writer threads finished at 1731422610147 (+1 ms)Closed recovered edits writer path=hdfs://localhost:40559/hbase/data/default/testSequentialEditLogSeqNum/ed6ff18d38ce2f69abd262db70dc19d1/recovered.edits/0000000000000000003-wal.1731422609945.temp (wrote 15 edits, skipped 0 edits in 0 ms) at 1731422610154 (+7 ms)Rename recovered edits hdfs://localhost:40559/hbase/data/default/testSequentialEditLogSeqNum/ed6ff18d38ce2f69abd262db70dc19d1/recovered.edits/0000000000000000003-wal.1731422609945.temp to hdfs://localhost:40559/hbase/data/default/testSequentialEditLogSeqNum/ed6ff18d38ce2f69abd262db70dc19d1/recovered.edits/0000000000000000020 at 1731422610155 (+1 ms)Processed 17 edits across 1 Regions in 36 ms; skipped=2; WAL=hdfs://localhost:40559/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1731422609936/wal.1731422609945, size=3.2 K, length=3256, corrupted=false, cancelled=false at 1731422610155 2024-11-12T14:43:30,167 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testSequentialEditLogSeqNum Thread=413 (was 408) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-425049348_22 at /127.0.0.1:44176 [Waiting for operation #15] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-425049348_22 at /127.0.0.1:38556 [Waiting for operation #9] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: sync.2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) - Thread LEAK? -, OpenFileDescriptor=963 (was 927) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=298 (was 298), ProcessCount=11 (was 11), AvailableMemoryMB=7417 (was 7420) 2024-11-12T14:43:30,179 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testRegionMadeOfBulkLoadedFilesOnly Thread=413, OpenFileDescriptor=963, MaxFileDescriptor=1048576, SystemLoadAverage=298, ProcessCount=11, AvailableMemoryMB=7417 2024-11-12T14:43:30,195 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-12T14:43:30,196 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-12T14:43:30,230 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-12T14:43:30,234 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-93246486, suffix=, logDir=hdfs://localhost:40559/hbase/WALs/hregion-93246486, archiveDir=hdfs://localhost:40559/hbase/oldWALs, maxLogs=32 2024-11-12T14:43:30,256 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-93246486/hregion-93246486.1731422610235, exclude list is [], retry=0 2024-11-12T14:43:30,259 DEBUG [AsyncFSWAL-17-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36175,DS-31e7ba0a-e37c-4fe6-8d2a-a4846cefa408,DISK] 2024-11-12T14:43:30,259 DEBUG [AsyncFSWAL-17-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45975,DS-80b047b6-34d2-41c0-99c3-5cd61ca40075,DISK] 2024-11-12T14:43:30,259 DEBUG [AsyncFSWAL-17-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37395,DS-7317e4d9-cef6-4b85-a947-106c3c917da9,DISK] 2024-11-12T14:43:30,285 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-93246486/hregion-93246486.1731422610235 2024-11-12T14:43:30,285 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36027:36027),(127.0.0.1/127.0.0.1:43643:43643),(127.0.0.1/127.0.0.1:40227:40227)] 2024-11-12T14:43:30,285 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => e413c0246e5665c0e9608535aac302c1, NAME => 'testRegionMadeOfBulkLoadedFilesOnly,,1731422610195.e413c0246e5665c0e9608535aac302c1.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testRegionMadeOfBulkLoadedFilesOnly', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40559/hbase 2024-11-12T14:43:30,293 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45975 is added to blk_1073741879_1056 (size=70) 2024-11-12T14:43:30,294 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36175 is added to blk_1073741879_1056 (size=70) 2024-11-12T14:43:30,294 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testRegionMadeOfBulkLoadedFilesOnly,,1731422610195.e413c0246e5665c0e9608535aac302c1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T14:43:30,295 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37395 is added to blk_1073741879_1056 (size=70) 2024-11-12T14:43:30,296 INFO [StoreOpener-e413c0246e5665c0e9608535aac302c1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region e413c0246e5665c0e9608535aac302c1 2024-11-12T14:43:30,297 INFO [StoreOpener-e413c0246e5665c0e9608535aac302c1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e413c0246e5665c0e9608535aac302c1 columnFamilyName a 2024-11-12T14:43:30,297 DEBUG [StoreOpener-e413c0246e5665c0e9608535aac302c1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:43:30,298 INFO [StoreOpener-e413c0246e5665c0e9608535aac302c1-1 {}] regionserver.HStore(327): Store=e413c0246e5665c0e9608535aac302c1/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T14:43:30,298 INFO [StoreOpener-e413c0246e5665c0e9608535aac302c1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region e413c0246e5665c0e9608535aac302c1 2024-11-12T14:43:30,299 INFO [StoreOpener-e413c0246e5665c0e9608535aac302c1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e413c0246e5665c0e9608535aac302c1 columnFamilyName b 2024-11-12T14:43:30,299 DEBUG [StoreOpener-e413c0246e5665c0e9608535aac302c1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:43:30,300 INFO [StoreOpener-e413c0246e5665c0e9608535aac302c1-1 {}] regionserver.HStore(327): Store=e413c0246e5665c0e9608535aac302c1/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T14:43:30,300 INFO [StoreOpener-e413c0246e5665c0e9608535aac302c1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region e413c0246e5665c0e9608535aac302c1 2024-11-12T14:43:30,301 INFO [StoreOpener-e413c0246e5665c0e9608535aac302c1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e413c0246e5665c0e9608535aac302c1 columnFamilyName c 2024-11-12T14:43:30,301 DEBUG [StoreOpener-e413c0246e5665c0e9608535aac302c1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:43:30,302 INFO [StoreOpener-e413c0246e5665c0e9608535aac302c1-1 {}] regionserver.HStore(327): Store=e413c0246e5665c0e9608535aac302c1/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T14:43:30,302 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for e413c0246e5665c0e9608535aac302c1 2024-11-12T14:43:30,302 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40559/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/e413c0246e5665c0e9608535aac302c1 2024-11-12T14:43:30,303 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40559/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/e413c0246e5665c0e9608535aac302c1 2024-11-12T14:43:30,304 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for e413c0246e5665c0e9608535aac302c1 2024-11-12T14:43:30,304 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for e413c0246e5665c0e9608535aac302c1 2024-11-12T14:43:30,304 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRegionMadeOfBulkLoadedFilesOnly descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-12T14:43:30,305 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for e413c0246e5665c0e9608535aac302c1 2024-11-12T14:43:30,307 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40559/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/e413c0246e5665c0e9608535aac302c1/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-12T14:43:30,308 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened e413c0246e5665c0e9608535aac302c1; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66591593, jitterRate=-0.007707938551902771}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-12T14:43:30,309 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for e413c0246e5665c0e9608535aac302c1: Writing region info on filesystem at 1731422610294Initializing all the Stores at 1731422610295 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731422610296 (+1 ms)Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731422610296Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731422610296Cleaning up temporary data from old regions at 1731422610304 (+8 ms)Region opened successfully at 1731422610309 (+5 ms) 2024-11-12T14:43:30,309 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing e413c0246e5665c0e9608535aac302c1, disabling compactions & flushes 2024-11-12T14:43:30,309 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testRegionMadeOfBulkLoadedFilesOnly,,1731422610195.e413c0246e5665c0e9608535aac302c1. 2024-11-12T14:43:30,309 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testRegionMadeOfBulkLoadedFilesOnly,,1731422610195.e413c0246e5665c0e9608535aac302c1. 2024-11-12T14:43:30,309 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testRegionMadeOfBulkLoadedFilesOnly,,1731422610195.e413c0246e5665c0e9608535aac302c1. after waiting 0 ms 2024-11-12T14:43:30,309 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testRegionMadeOfBulkLoadedFilesOnly,,1731422610195.e413c0246e5665c0e9608535aac302c1. 2024-11-12T14:43:30,310 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testRegionMadeOfBulkLoadedFilesOnly,,1731422610195.e413c0246e5665c0e9608535aac302c1. 2024-11-12T14:43:30,310 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for e413c0246e5665c0e9608535aac302c1: Waiting for close lock at 1731422610309Disabling compacts and flushes for region at 1731422610309Disabling writes for close at 1731422610309Writing region close event to WAL at 1731422610310 (+1 ms)Closed at 1731422610310 2024-11-12T14:43:30,313 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /hbase/WALs/hregion-93246486/hregion-93246486.1731422610235 not finished, retry = 0 2024-11-12T14:43:30,313 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45975 is added to blk_1073741878_1055 (size=175) 2024-11-12T14:43:30,313 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36175 is added to blk_1073741878_1055 (size=175) 2024-11-12T14:43:30,314 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37395 is added to blk_1073741878_1055 (size=175) 2024-11-12T14:43:30,418 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-12T14:43:30,418 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-93246486:(num 1731422610235) 2024-11-12T14:43:30,418 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-12T14:43:30,421 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:40559/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731422610194, archiveDir=hdfs://localhost:40559/hbase/oldWALs, maxLogs=32 2024-11-12T14:43:30,437 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731422610194/wal.1731422610422, exclude list is [], retry=0 2024-11-12T14:43:30,440 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45975,DS-80b047b6-34d2-41c0-99c3-5cd61ca40075,DISK] 2024-11-12T14:43:30,440 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37395,DS-7317e4d9-cef6-4b85-a947-106c3c917da9,DISK] 2024-11-12T14:43:30,441 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36175,DS-31e7ba0a-e37c-4fe6-8d2a-a4846cefa408,DISK] 2024-11-12T14:43:30,464 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731422610194/wal.1731422610422 2024-11-12T14:43:30,464 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43643:43643),(127.0.0.1/127.0.0.1:40227:40227),(127.0.0.1/127.0.0.1:36027:36027)] 2024-11-12T14:43:30,464 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => e413c0246e5665c0e9608535aac302c1, NAME => 'testRegionMadeOfBulkLoadedFilesOnly,,1731422610195.e413c0246e5665c0e9608535aac302c1.', STARTKEY => '', ENDKEY => ''} 2024-11-12T14:43:30,464 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testRegionMadeOfBulkLoadedFilesOnly,,1731422610195.e413c0246e5665c0e9608535aac302c1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T14:43:30,465 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for e413c0246e5665c0e9608535aac302c1 2024-11-12T14:43:30,465 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for e413c0246e5665c0e9608535aac302c1 2024-11-12T14:43:30,466 INFO [StoreOpener-e413c0246e5665c0e9608535aac302c1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region e413c0246e5665c0e9608535aac302c1 2024-11-12T14:43:30,467 INFO [StoreOpener-e413c0246e5665c0e9608535aac302c1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e413c0246e5665c0e9608535aac302c1 columnFamilyName a 2024-11-12T14:43:30,467 DEBUG [StoreOpener-e413c0246e5665c0e9608535aac302c1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:43:30,468 INFO [StoreOpener-e413c0246e5665c0e9608535aac302c1-1 {}] regionserver.HStore(327): Store=e413c0246e5665c0e9608535aac302c1/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T14:43:30,468 INFO [StoreOpener-e413c0246e5665c0e9608535aac302c1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region e413c0246e5665c0e9608535aac302c1 2024-11-12T14:43:30,469 INFO [StoreOpener-e413c0246e5665c0e9608535aac302c1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e413c0246e5665c0e9608535aac302c1 columnFamilyName b 2024-11-12T14:43:30,469 DEBUG [StoreOpener-e413c0246e5665c0e9608535aac302c1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:43:30,470 INFO [StoreOpener-e413c0246e5665c0e9608535aac302c1-1 {}] regionserver.HStore(327): Store=e413c0246e5665c0e9608535aac302c1/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T14:43:30,470 INFO [StoreOpener-e413c0246e5665c0e9608535aac302c1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region e413c0246e5665c0e9608535aac302c1 2024-11-12T14:43:30,470 INFO [StoreOpener-e413c0246e5665c0e9608535aac302c1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e413c0246e5665c0e9608535aac302c1 columnFamilyName c 2024-11-12T14:43:30,471 DEBUG [StoreOpener-e413c0246e5665c0e9608535aac302c1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:43:30,471 INFO [StoreOpener-e413c0246e5665c0e9608535aac302c1-1 {}] regionserver.HStore(327): Store=e413c0246e5665c0e9608535aac302c1/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T14:43:30,471 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for e413c0246e5665c0e9608535aac302c1 2024-11-12T14:43:30,472 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40559/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/e413c0246e5665c0e9608535aac302c1 2024-11-12T14:43:30,473 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40559/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/e413c0246e5665c0e9608535aac302c1 2024-11-12T14:43:30,474 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for e413c0246e5665c0e9608535aac302c1 2024-11-12T14:43:30,474 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for e413c0246e5665c0e9608535aac302c1 2024-11-12T14:43:30,475 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRegionMadeOfBulkLoadedFilesOnly descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-12T14:43:30,476 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for e413c0246e5665c0e9608535aac302c1 2024-11-12T14:43:30,477 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened e413c0246e5665c0e9608535aac302c1; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66324135, jitterRate=-0.011693373322486877}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-12T14:43:30,478 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for e413c0246e5665c0e9608535aac302c1: Writing region info on filesystem at 1731422610465Initializing all the Stores at 1731422610466 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731422610466Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731422610466Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731422610466Cleaning up temporary data from old regions at 1731422610475 (+9 ms)Region opened successfully at 1731422610478 (+3 ms) 2024-11-12T14:43:30,482 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40559/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile is 28, key is \x0D/a:a/1731422610481/Put/seqid=0 2024-11-12T14:43:30,490 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37395 is added to blk_1073741881_1058 (size=4826) 2024-11-12T14:43:30,491 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36175 is added to blk_1073741881_1058 (size=4826) 2024-11-12T14:43:30,491 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45975 is added to blk_1073741881_1058 (size=4826) 2024-11-12T14:43:30,491 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:40559/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile for inclusion in e413c0246e5665c0e9608535aac302c1/a 2024-11-12T14:43:30,498 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first= last=z 2024-11-12T14:43:30,498 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-11-12T14:43:30,499 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for e413c0246e5665c0e9608535aac302c1: 2024-11-12T14:43:30,500 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40559/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile as hdfs://localhost:40559/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/e413c0246e5665c0e9608535aac302c1/a/8b7d4aa4487243d5a6a6dee9d2977e89_SeqId_3_ 2024-11-12T14:43:30,501 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:40559/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile into e413c0246e5665c0e9608535aac302c1/a as hdfs://localhost:40559/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/e413c0246e5665c0e9608535aac302c1/a/8b7d4aa4487243d5a6a6dee9d2977e89_SeqId_3_ - updating store file list. 2024-11-12T14:43:30,507 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for 8b7d4aa4487243d5a6a6dee9d2977e89_SeqId_3_: NONE, but ROW specified in column family configuration 2024-11-12T14:43:30,507 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:40559/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/e413c0246e5665c0e9608535aac302c1/a/8b7d4aa4487243d5a6a6dee9d2977e89_SeqId_3_ into e413c0246e5665c0e9608535aac302c1/a 2024-11-12T14:43:30,507 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:40559/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile into e413c0246e5665c0e9608535aac302c1/a (new location: hdfs://localhost:40559/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/e413c0246e5665c0e9608535aac302c1/a/8b7d4aa4487243d5a6a6dee9d2977e89_SeqId_3_) 2024-11-12T14:43:30,546 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:40559/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731422610194/wal.1731422610422, size=0 (0bytes) 2024-11-12T14:43:30,546 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:40559/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731422610194/wal.1731422610422 might be still open, length is 0 2024-11-12T14:43:30,547 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:40559/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731422610194/wal.1731422610422 2024-11-12T14:43:30,547 WARN [IPC Server handler 0 on default port 40559 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731422610194/wal.1731422610422 has not been closed. Lease recovery is in progress. RecoveryId = 1059 for block blk_1073741880_1057 2024-11-12T14:43:30,547 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:40559/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731422610194/wal.1731422610422 after 0ms 2024-11-12T14:43:33,029 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-425049348_22 at /127.0.0.1:38644 [Receiving block BP-1980882744-172.17.0.2-1731422596004:blk_1073741880_1057] {}] datanode.DataXceiver(331): 127.0.0.1:45975:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:38644 dst: /127.0.0.1:45975 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:45975 remote=/127.0.0.1:38644]. Total timeout mills is 60000, 57487 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:43:33,030 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-425049348_22 at /127.0.0.1:36164 [Receiving block BP-1980882744-172.17.0.2-1731422596004:blk_1073741880_1057] {}] datanode.DataXceiver(331): 127.0.0.1:36175:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:36164 dst: /127.0.0.1:36175 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:43:33,030 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-425049348_22 at /127.0.0.1:44306 [Receiving block BP-1980882744-172.17.0.2-1731422596004:blk_1073741880_1057] {}] datanode.DataXceiver(331): 127.0.0.1:37395:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:44306 dst: /127.0.0.1:37395 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:43:33,036 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45975 is added to blk_1073741880_1059 (size=606) 2024-11-12T14:43:33,037 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37395 is added to blk_1073741880_1059 (size=606) 2024-11-12T14:43:33,038 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36175 is added to blk_1073741880_1059 (size=606) 2024-11-12T14:43:34,548 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:40559/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731422610194/wal.1731422610422 after 4001ms 2024-11-12T14:43:34,579 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-12T14:43:34,579 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:40559/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731422610194/wal.1731422610422 took 4033ms 2024-11-12T14:43:34,581 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:40559/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731422610194/wal.1731422610422; continuing. 2024-11-12T14:43:34,581 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:40559/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731422610194/wal.1731422610422 so closing down 2024-11-12T14:43:34,582 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-12T14:43:34,583 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000005-wal.1731422610422.temp 2024-11-12T14:43:34,607 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:40559/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/e413c0246e5665c0e9608535aac302c1/recovered.edits/0000000000000000005-wal.1731422610422.temp 2024-11-12T14:43:34,607 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-12T14:43:34,613 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45975 is added to blk_1073741882_1060 (size=329) 2024-11-12T14:43:34,613 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37395 is added to blk_1073741882_1060 (size=329) 2024-11-12T14:43:34,614 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36175 is added to blk_1073741882_1060 (size=329) 2024-11-12T14:43:34,614 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:40559/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/e413c0246e5665c0e9608535aac302c1/recovered.edits/0000000000000000005-wal.1731422610422.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-11-12T14:43:34,615 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:40559/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/e413c0246e5665c0e9608535aac302c1/recovered.edits/0000000000000000005-wal.1731422610422.temp to hdfs://localhost:40559/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/e413c0246e5665c0e9608535aac302c1/recovered.edits/0000000000000000005 2024-11-12T14:43:34,616 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 2 edits across 1 Regions in 35 ms; skipped=1; WAL=hdfs://localhost:40559/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731422610194/wal.1731422610422, size=0, length=0, corrupted=false, cancelled=false 2024-11-12T14:43:34,616 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:40559/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731422610194/wal.1731422610422, journal: Splitting hdfs://localhost:40559/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731422610194/wal.1731422610422, size=0 (0bytes) at 1731422610546Finishing writing output for hdfs://localhost:40559/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731422610194/wal.1731422610422 so closing down at 1731422614581 (+4035 ms)Creating recovered edits writer path=hdfs://localhost:40559/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/e413c0246e5665c0e9608535aac302c1/recovered.edits/0000000000000000005-wal.1731422610422.temp at 1731422614607 (+26 ms)3 split writer threads finished at 1731422614607Closed recovered edits writer path=hdfs://localhost:40559/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/e413c0246e5665c0e9608535aac302c1/recovered.edits/0000000000000000005-wal.1731422610422.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1731422614614 (+7 ms)Rename recovered edits hdfs://localhost:40559/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/e413c0246e5665c0e9608535aac302c1/recovered.edits/0000000000000000005-wal.1731422610422.temp to hdfs://localhost:40559/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/e413c0246e5665c0e9608535aac302c1/recovered.edits/0000000000000000005 at 1731422614615 (+1 ms)Processed 2 edits across 1 Regions in 35 ms; skipped=1; WAL=hdfs://localhost:40559/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731422610194/wal.1731422610422, size=0, length=0, corrupted=false, cancelled=false at 1731422614616 (+1 ms) 2024-11-12T14:43:34,617 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:40559/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731422610194/wal.1731422610422 to hdfs://localhost:40559/hbase/oldWALs/wal.1731422610422 2024-11-12T14:43:34,618 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:40559/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/e413c0246e5665c0e9608535aac302c1/recovered.edits/0000000000000000005 2024-11-12T14:43:34,618 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-12T14:43:34,620 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:40559/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731422610194, archiveDir=hdfs://localhost:40559/hbase/oldWALs, maxLogs=32 2024-11-12T14:43:34,634 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731422610194/wal.1731422614621, exclude list is [], retry=0 2024-11-12T14:43:34,637 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36175,DS-31e7ba0a-e37c-4fe6-8d2a-a4846cefa408,DISK] 2024-11-12T14:43:34,637 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45975,DS-80b047b6-34d2-41c0-99c3-5cd61ca40075,DISK] 2024-11-12T14:43:34,637 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37395,DS-7317e4d9-cef6-4b85-a947-106c3c917da9,DISK] 2024-11-12T14:43:34,661 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731422610194/wal.1731422614621 2024-11-12T14:43:34,661 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36027:36027),(127.0.0.1/127.0.0.1:43643:43643),(127.0.0.1/127.0.0.1:40227:40227)] 2024-11-12T14:43:34,661 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => e413c0246e5665c0e9608535aac302c1, NAME => 'testRegionMadeOfBulkLoadedFilesOnly,,1731422610195.e413c0246e5665c0e9608535aac302c1.', STARTKEY => '', ENDKEY => ''} 2024-11-12T14:43:34,661 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testRegionMadeOfBulkLoadedFilesOnly,,1731422610195.e413c0246e5665c0e9608535aac302c1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T14:43:34,662 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for e413c0246e5665c0e9608535aac302c1 2024-11-12T14:43:34,662 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for e413c0246e5665c0e9608535aac302c1 2024-11-12T14:43:34,663 INFO [StoreOpener-e413c0246e5665c0e9608535aac302c1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region e413c0246e5665c0e9608535aac302c1 2024-11-12T14:43:34,664 INFO [StoreOpener-e413c0246e5665c0e9608535aac302c1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e413c0246e5665c0e9608535aac302c1 columnFamilyName a 2024-11-12T14:43:34,664 DEBUG [StoreOpener-e413c0246e5665c0e9608535aac302c1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:43:34,670 DEBUG [StoreFileOpener-e413c0246e5665c0e9608535aac302c1-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for 8b7d4aa4487243d5a6a6dee9d2977e89_SeqId_3_: NONE, but ROW specified in column family configuration 2024-11-12T14:43:34,671 DEBUG [StoreOpener-e413c0246e5665c0e9608535aac302c1-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:40559/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/e413c0246e5665c0e9608535aac302c1/a/8b7d4aa4487243d5a6a6dee9d2977e89_SeqId_3_ 2024-11-12T14:43:34,671 INFO [StoreOpener-e413c0246e5665c0e9608535aac302c1-1 {}] regionserver.HStore(327): Store=e413c0246e5665c0e9608535aac302c1/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T14:43:34,671 INFO [StoreOpener-e413c0246e5665c0e9608535aac302c1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region e413c0246e5665c0e9608535aac302c1 2024-11-12T14:43:34,672 INFO [StoreOpener-e413c0246e5665c0e9608535aac302c1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e413c0246e5665c0e9608535aac302c1 columnFamilyName b 2024-11-12T14:43:34,672 DEBUG [StoreOpener-e413c0246e5665c0e9608535aac302c1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:43:34,673 INFO [StoreOpener-e413c0246e5665c0e9608535aac302c1-1 {}] regionserver.HStore(327): Store=e413c0246e5665c0e9608535aac302c1/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T14:43:34,673 INFO [StoreOpener-e413c0246e5665c0e9608535aac302c1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region e413c0246e5665c0e9608535aac302c1 2024-11-12T14:43:34,674 INFO [StoreOpener-e413c0246e5665c0e9608535aac302c1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e413c0246e5665c0e9608535aac302c1 columnFamilyName c 2024-11-12T14:43:34,674 DEBUG [StoreOpener-e413c0246e5665c0e9608535aac302c1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:43:34,674 INFO [StoreOpener-e413c0246e5665c0e9608535aac302c1-1 {}] regionserver.HStore(327): Store=e413c0246e5665c0e9608535aac302c1/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T14:43:34,675 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for e413c0246e5665c0e9608535aac302c1 2024-11-12T14:43:34,675 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40559/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/e413c0246e5665c0e9608535aac302c1 2024-11-12T14:43:34,677 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:40559/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/e413c0246e5665c0e9608535aac302c1 2024-11-12T14:43:34,678 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:40559/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/e413c0246e5665c0e9608535aac302c1/recovered.edits/0000000000000000005 2024-11-12T14:43:34,703 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-12T14:43:34,704 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 1, skipped 0, firstSequenceIdInLog=5, maxSequenceIdInLog=5, path=hdfs://localhost:40559/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/e413c0246e5665c0e9608535aac302c1/recovered.edits/0000000000000000005 2024-11-12T14:43:34,704 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing e413c0246e5665c0e9608535aac302c1 3/3 column families, dataSize=58 B heapSize=904 B 2024-11-12T14:43:34,717 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40559/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/e413c0246e5665c0e9608535aac302c1/.tmp/a/290efd2fd87e4d65b29d2bb29b2a4d06 is 62, key is testRegionMadeOfBulkLoadedFilesOnly/a:a/1731422610513/Put/seqid=0 2024-11-12T14:43:34,723 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36175 is added to blk_1073741884_1062 (size=5149) 2024-11-12T14:43:34,723 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37395 is added to blk_1073741884_1062 (size=5149) 2024-11-12T14:43:34,723 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45975 is added to blk_1073741884_1062 (size=5149) 2024-11-12T14:43:34,724 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58 B at sequenceid=5 (bloomFilter=true), to=hdfs://localhost:40559/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/e413c0246e5665c0e9608535aac302c1/.tmp/a/290efd2fd87e4d65b29d2bb29b2a4d06 2024-11-12T14:43:34,729 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40559/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/e413c0246e5665c0e9608535aac302c1/.tmp/a/290efd2fd87e4d65b29d2bb29b2a4d06 as hdfs://localhost:40559/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/e413c0246e5665c0e9608535aac302c1/a/290efd2fd87e4d65b29d2bb29b2a4d06 2024-11-12T14:43:34,734 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40559/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/e413c0246e5665c0e9608535aac302c1/a/290efd2fd87e4d65b29d2bb29b2a4d06, entries=1, sequenceid=5, filesize=5.0 K 2024-11-12T14:43:34,734 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~58 B/58, heapSize ~376 B/376, currentSize=0 B/0 for e413c0246e5665c0e9608535aac302c1 in 30ms, sequenceid=5, compaction requested=false; wal=null 2024-11-12T14:43:34,735 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:40559/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/e413c0246e5665c0e9608535aac302c1/recovered.edits/0000000000000000005 2024-11-12T14:43:34,736 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for e413c0246e5665c0e9608535aac302c1 2024-11-12T14:43:34,736 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for e413c0246e5665c0e9608535aac302c1 2024-11-12T14:43:34,737 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRegionMadeOfBulkLoadedFilesOnly descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-12T14:43:34,739 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for e413c0246e5665c0e9608535aac302c1 2024-11-12T14:43:34,741 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40559/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/e413c0246e5665c0e9608535aac302c1/recovered.edits/5.seqid, newMaxSeqId=5, maxSeqId=1 2024-11-12T14:43:34,742 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened e413c0246e5665c0e9608535aac302c1; next sequenceid=6; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68994962, jitterRate=0.028105050325393677}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-12T14:43:34,742 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for e413c0246e5665c0e9608535aac302c1: Writing region info on filesystem at 1731422614662Initializing all the Stores at 1731422614663 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731422614663Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731422614663Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731422614663Obtaining lock to block concurrent updates at 1731422614704 (+41 ms)Preparing flush snapshotting stores in e413c0246e5665c0e9608535aac302c1 at 1731422614704Finished memstore snapshotting testRegionMadeOfBulkLoadedFilesOnly,,1731422610195.e413c0246e5665c0e9608535aac302c1., syncing WAL and waiting on mvcc, flushsize=dataSize=58, getHeapSize=856, getOffHeapSize=0, getCellsCount=1 at 1731422614704Flushing stores of testRegionMadeOfBulkLoadedFilesOnly,,1731422610195.e413c0246e5665c0e9608535aac302c1. at 1731422614704Flushing e413c0246e5665c0e9608535aac302c1/a: creating writer at 1731422614705 (+1 ms)Flushing e413c0246e5665c0e9608535aac302c1/a: appending metadata at 1731422614717 (+12 ms)Flushing e413c0246e5665c0e9608535aac302c1/a: closing flushed file at 1731422614717Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@756df4ed: reopening flushed file at 1731422614728 (+11 ms)Finished flush of dataSize ~58 B/58, heapSize ~376 B/376, currentSize=0 B/0 for e413c0246e5665c0e9608535aac302c1 in 30ms, sequenceid=5, compaction requested=false; wal=null at 1731422614734 (+6 ms)Cleaning up temporary data from old regions at 1731422614736 (+2 ms)Region opened successfully at 1731422614742 (+6 ms) 2024-11-12T14:43:34,746 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing e413c0246e5665c0e9608535aac302c1, disabling compactions & flushes 2024-11-12T14:43:34,746 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testRegionMadeOfBulkLoadedFilesOnly,,1731422610195.e413c0246e5665c0e9608535aac302c1. 2024-11-12T14:43:34,746 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testRegionMadeOfBulkLoadedFilesOnly,,1731422610195.e413c0246e5665c0e9608535aac302c1. 2024-11-12T14:43:34,746 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testRegionMadeOfBulkLoadedFilesOnly,,1731422610195.e413c0246e5665c0e9608535aac302c1. after waiting 0 ms 2024-11-12T14:43:34,746 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testRegionMadeOfBulkLoadedFilesOnly,,1731422610195.e413c0246e5665c0e9608535aac302c1. 2024-11-12T14:43:34,747 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testRegionMadeOfBulkLoadedFilesOnly,,1731422610195.e413c0246e5665c0e9608535aac302c1. 2024-11-12T14:43:34,747 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for e413c0246e5665c0e9608535aac302c1: Waiting for close lock at 1731422614746Disabling compacts and flushes for region at 1731422614746Disabling writes for close at 1731422614746Writing region close event to WAL at 1731422614747 (+1 ms)Closed at 1731422614747 2024-11-12T14:43:34,750 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45975 is added to blk_1073741883_1061 (size=175) 2024-11-12T14:43:34,750 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36175 is added to blk_1073741883_1061 (size=175) 2024-11-12T14:43:34,750 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37395 is added to blk_1073741883_1061 (size=175) 2024-11-12T14:43:34,752 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-12T14:43:34,752 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1731422614621) 2024-11-12T14:43:34,767 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testRegionMadeOfBulkLoadedFilesOnly Thread=417 (was 413) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1935351217_22 at /127.0.0.1:38682 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5a5e6c5c-3b82-c3b3-1527-3525a941b154/cluster_22b4e46d-f388-f2af-b32c-f9fd886af812/data/data1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-17-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1935351217_22 at /127.0.0.1:36170 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-17-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5a5e6c5c-3b82-c3b3-1527-3525a941b154/cluster_22b4e46d-f388-f2af-b32c-f9fd886af812/data/data2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1935351217_22 at /127.0.0.1:44350 [Waiting for operation #3] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-17-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkinstestRegionMadeOfBulkLoadedFilesOnly@localhost:40559 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1685805399) connection to localhost/127.0.0.1:40559 from jenkinstestRegionMadeOfBulkLoadedFilesOnly java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) - Thread LEAK? -, OpenFileDescriptor=1023 (was 963) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=283 (was 298), ProcessCount=11 (was 11), AvailableMemoryMB=7423 (was 7417) - AvailableMemoryMB LEAK? - 2024-11-12T14:43:34,779 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsAfterRegionMovedWithMultiCF Thread=417, OpenFileDescriptor=1023, MaxFileDescriptor=1048576, SystemLoadAverage=283, ProcessCount=11, AvailableMemoryMB=7422 2024-11-12T14:43:34,794 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-12T14:43:34,798 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-12T14:43:34,802 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 3945e1ca6f47,34233,1731422599017 2024-11-12T14:43:34,804 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@6249fb8e 2024-11-12T14:43:34,805 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-12T14:43:34,807 INFO [HMaster-EventLoopGroup-2-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60872, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-12T14:43:34,810 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34233 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testReplayEditsAfterRegionMovedWithMultiCF', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-12T14:43:34,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34233 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF 2024-11-12T14:43:34,818 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_PRE_OPERATION 2024-11-12T14:43:34,820 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34233 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testReplayEditsAfterRegionMovedWithMultiCF" procId is: 4 2024-11-12T14:43:34,820 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:43:34,822 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-12T14:43:34,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-12T14:43:34,833 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36175 is added to blk_1073741885_1063 (size=694) 2024-11-12T14:43:34,833 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45975 is added to blk_1073741885_1063 (size=694) 2024-11-12T14:43:34,833 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37395 is added to blk_1073741885_1063 (size=694) 2024-11-12T14:43:34,836 INFO [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 06dfc67af613562882ee0461d2aa7115, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1731422614808.06dfc67af613562882ee0461d2aa7115.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsAfterRegionMovedWithMultiCF', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40559/user/jenkins/test-data/41a388f8-1764-825e-e285-25964d37c9f3 2024-11-12T14:43:34,843 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45975 is added to blk_1073741886_1064 (size=77) 2024-11-12T14:43:34,843 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36175 is added to blk_1073741886_1064 (size=77) 2024-11-12T14:43:34,844 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37395 is added to blk_1073741886_1064 (size=77) 2024-11-12T14:43:34,844 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1731422614808.06dfc67af613562882ee0461d2aa7115.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T14:43:34,844 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1722): Closing 06dfc67af613562882ee0461d2aa7115, disabling compactions & flushes 2024-11-12T14:43:34,844 INFO [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1731422614808.06dfc67af613562882ee0461d2aa7115. 2024-11-12T14:43:34,844 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731422614808.06dfc67af613562882ee0461d2aa7115. 2024-11-12T14:43:34,844 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731422614808.06dfc67af613562882ee0461d2aa7115. after waiting 0 ms 2024-11-12T14:43:34,844 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1731422614808.06dfc67af613562882ee0461d2aa7115. 2024-11-12T14:43:34,845 INFO [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1731422614808.06dfc67af613562882ee0461d2aa7115. 2024-11-12T14:43:34,845 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1676): Region close journal for 06dfc67af613562882ee0461d2aa7115: Waiting for close lock at 1731422614844Disabling compacts and flushes for region at 1731422614844Disabling writes for close at 1731422614844Writing region close event to WAL at 1731422614844Closed at 1731422614844 2024-11-12T14:43:34,846 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_ADD_TO_META 2024-11-12T14:43:34,852 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testReplayEditsAfterRegionMovedWithMultiCF,,1731422614808.06dfc67af613562882ee0461d2aa7115.","families":{"info":[{"qualifier":"regioninfo","vlen":76,"tag":[],"timestamp":"1731422614847"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731422614847"}]},"ts":"1731422614847"} 2024-11-12T14:43:34,855 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-12T14:43:34,857 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-12T14:43:34,859 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testReplayEditsAfterRegionMovedWithMultiCF","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731422614857"}]},"ts":"1731422614857"} 2024-11-12T14:43:34,862 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testReplayEditsAfterRegionMovedWithMultiCF, state=ENABLING in hbase:meta 2024-11-12T14:43:34,863 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {3945e1ca6f47=0} racks are {/default-rack=0} 2024-11-12T14:43:34,864 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-12T14:43:34,864 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-12T14:43:34,864 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-11-12T14:43:34,864 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-12T14:43:34,864 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-12T14:43:34,864 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-11-12T14:43:34,864 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-12T14:43:34,864 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-12T14:43:34,864 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-11-12T14:43:34,864 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-12T14:43:34,866 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=06dfc67af613562882ee0461d2aa7115, ASSIGN}] 2024-11-12T14:43:34,868 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=06dfc67af613562882ee0461d2aa7115, ASSIGN 2024-11-12T14:43:34,869 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=06dfc67af613562882ee0461d2aa7115, ASSIGN; state=OFFLINE, location=3945e1ca6f47,38727,1731422599737; forceNewPlan=false, retain=false 2024-11-12T14:43:34,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-12T14:43:35,022 INFO [3945e1ca6f47:34233 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-12T14:43:35,022 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=06dfc67af613562882ee0461d2aa7115, regionState=OPENING, regionLocation=3945e1ca6f47,38727,1731422599737 2024-11-12T14:43:35,026 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=06dfc67af613562882ee0461d2aa7115, ASSIGN because future has completed 2024-11-12T14:43:35,026 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 06dfc67af613562882ee0461d2aa7115, server=3945e1ca6f47,38727,1731422599737}] 2024-11-12T14:43:35,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-12T14:43:35,180 DEBUG [RSProcedureDispatcher-pool-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-12T14:43:35,182 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49507, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-12T14:43:35,190 INFO [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1731422614808.06dfc67af613562882ee0461d2aa7115. 2024-11-12T14:43:35,190 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 06dfc67af613562882ee0461d2aa7115, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1731422614808.06dfc67af613562882ee0461d2aa7115.', STARTKEY => '', ENDKEY => ''} 2024-11-12T14:43:35,191 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF 06dfc67af613562882ee0461d2aa7115 2024-11-12T14:43:35,191 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1731422614808.06dfc67af613562882ee0461d2aa7115.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T14:43:35,191 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 06dfc67af613562882ee0461d2aa7115 2024-11-12T14:43:35,191 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 06dfc67af613562882ee0461d2aa7115 2024-11-12T14:43:35,193 INFO [StoreOpener-06dfc67af613562882ee0461d2aa7115-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 06dfc67af613562882ee0461d2aa7115 2024-11-12T14:43:35,195 INFO [StoreOpener-06dfc67af613562882ee0461d2aa7115-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 06dfc67af613562882ee0461d2aa7115 columnFamilyName cf1 2024-11-12T14:43:35,195 DEBUG [StoreOpener-06dfc67af613562882ee0461d2aa7115-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:43:35,196 INFO [StoreOpener-06dfc67af613562882ee0461d2aa7115-1 {}] regionserver.HStore(327): Store=06dfc67af613562882ee0461d2aa7115/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T14:43:35,196 INFO [StoreOpener-06dfc67af613562882ee0461d2aa7115-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 06dfc67af613562882ee0461d2aa7115 2024-11-12T14:43:35,198 INFO [StoreOpener-06dfc67af613562882ee0461d2aa7115-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 06dfc67af613562882ee0461d2aa7115 columnFamilyName cf2 2024-11-12T14:43:35,198 DEBUG [StoreOpener-06dfc67af613562882ee0461d2aa7115-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:43:35,199 INFO [StoreOpener-06dfc67af613562882ee0461d2aa7115-1 {}] regionserver.HStore(327): Store=06dfc67af613562882ee0461d2aa7115/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T14:43:35,199 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 06dfc67af613562882ee0461d2aa7115 2024-11-12T14:43:35,200 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40559/user/jenkins/test-data/41a388f8-1764-825e-e285-25964d37c9f3/data/default/testReplayEditsAfterRegionMovedWithMultiCF/06dfc67af613562882ee0461d2aa7115 2024-11-12T14:43:35,201 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40559/user/jenkins/test-data/41a388f8-1764-825e-e285-25964d37c9f3/data/default/testReplayEditsAfterRegionMovedWithMultiCF/06dfc67af613562882ee0461d2aa7115 2024-11-12T14:43:35,202 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 06dfc67af613562882ee0461d2aa7115 2024-11-12T14:43:35,202 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 06dfc67af613562882ee0461d2aa7115 2024-11-12T14:43:35,203 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-11-12T14:43:35,205 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 06dfc67af613562882ee0461d2aa7115 2024-11-12T14:43:35,207 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40559/user/jenkins/test-data/41a388f8-1764-825e-e285-25964d37c9f3/data/default/testReplayEditsAfterRegionMovedWithMultiCF/06dfc67af613562882ee0461d2aa7115/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-12T14:43:35,208 INFO [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 06dfc67af613562882ee0461d2aa7115; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61803183, jitterRate=-0.07906080782413483}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-11-12T14:43:35,208 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 06dfc67af613562882ee0461d2aa7115 2024-11-12T14:43:35,208 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 06dfc67af613562882ee0461d2aa7115: Running coprocessor pre-open hook at 1731422615191Writing region info on filesystem at 1731422615191Initializing all the Stores at 1731422615193 (+2 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731422615193Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731422615193Cleaning up temporary data from old regions at 1731422615202 (+9 ms)Running coprocessor post-open hooks at 1731422615208 (+6 ms)Region opened successfully at 1731422615208 2024-11-12T14:43:35,211 INFO [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1731422614808.06dfc67af613562882ee0461d2aa7115., pid=6, masterSystemTime=1731422615179 2024-11-12T14:43:35,213 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1731422614808.06dfc67af613562882ee0461d2aa7115. 2024-11-12T14:43:35,214 INFO [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1731422614808.06dfc67af613562882ee0461d2aa7115. 2024-11-12T14:43:35,215 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=06dfc67af613562882ee0461d2aa7115, regionState=OPEN, openSeqNum=2, regionLocation=3945e1ca6f47,38727,1731422599737 2024-11-12T14:43:35,218 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 06dfc67af613562882ee0461d2aa7115, server=3945e1ca6f47,38727,1731422599737 because future has completed 2024-11-12T14:43:35,223 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-12T14:43:35,225 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 06dfc67af613562882ee0461d2aa7115, server=3945e1ca6f47,38727,1731422599737 in 194 msec 2024-11-12T14:43:35,227 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-12T14:43:35,227 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=06dfc67af613562882ee0461d2aa7115, ASSIGN in 358 msec 2024-11-12T14:43:35,228 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-12T14:43:35,228 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testReplayEditsAfterRegionMovedWithMultiCF","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731422615228"}]},"ts":"1731422615228"} 2024-11-12T14:43:35,230 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testReplayEditsAfterRegionMovedWithMultiCF, state=ENABLED in hbase:meta 2024-11-12T14:43:35,232 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_POST_OPERATION 2024-11-12T14:43:35,234 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF in 420 msec 2024-11-12T14:43:35,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-12T14:43:35,456 INFO [RPCClient-NioEventLoopGroup-6-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testReplayEditsAfterRegionMovedWithMultiCF completed 2024-11-12T14:43:35,457 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testReplayEditsAfterRegionMovedWithMultiCF get assigned. Timeout = 60000ms 2024-11-12T14:43:35,460 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-12T14:43:35,468 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testReplayEditsAfterRegionMovedWithMultiCF assigned to meta. Checking AM states. 2024-11-12T14:43:35,469 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-12T14:43:35,470 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testReplayEditsAfterRegionMovedWithMultiCF assigned. 2024-11-12T14:43:35,482 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1731422614808.06dfc67af613562882ee0461d2aa7115., hostname=3945e1ca6f47,38727,1731422599737, seqNum=2] 2024-11-12T14:43:35,483 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T14:43:35,485 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59246, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T14:43:35,502 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34233 {}] master.HMaster(2410): Client=jenkins//172.17.0.2 move hri=06dfc67af613562882ee0461d2aa7115, source=3945e1ca6f47,38727,1731422599737, destination=3945e1ca6f47,41431,1731422599613, warming up region on 3945e1ca6f47,41431,1731422599613 2024-11-12T14:43:35,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34233 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-12T14:43:35,504 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34233 {}] master.HMaster(2414): Client=jenkins//172.17.0.2 move hri=06dfc67af613562882ee0461d2aa7115, source=3945e1ca6f47,38727,1731422599737, destination=3945e1ca6f47,41431,1731422599613, running balancer 2024-11-12T14:43:35,505 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55721, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-12T14:43:35,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34233 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=06dfc67af613562882ee0461d2aa7115, REOPEN/MOVE 2024-11-12T14:43:35,506 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=06dfc67af613562882ee0461d2aa7115, REOPEN/MOVE 2024-11-12T14:43:35,508 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=06dfc67af613562882ee0461d2aa7115, regionState=CLOSING, regionLocation=3945e1ca6f47,38727,1731422599737 2024-11-12T14:43:35,510 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41431 {}] regionserver.RSRpcServices(2066): Warmup testReplayEditsAfterRegionMovedWithMultiCF,,1731422614808.06dfc67af613562882ee0461d2aa7115. 2024-11-12T14:43:35,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41431 {}] regionserver.HRegion(7855): Warmup {ENCODED => 06dfc67af613562882ee0461d2aa7115, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1731422614808.06dfc67af613562882ee0461d2aa7115.', STARTKEY => '', ENDKEY => ''} 2024-11-12T14:43:35,510 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=06dfc67af613562882ee0461d2aa7115, REOPEN/MOVE because future has completed 2024-11-12T14:43:35,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41431 {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1731422614808.06dfc67af613562882ee0461d2aa7115.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T14:43:35,510 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-12T14:43:35,511 INFO [StoreOpener-06dfc67af613562882ee0461d2aa7115-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 06dfc67af613562882ee0461d2aa7115 2024-11-12T14:43:35,511 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure 06dfc67af613562882ee0461d2aa7115, server=3945e1ca6f47,38727,1731422599737}] 2024-11-12T14:43:35,512 INFO [StoreOpener-06dfc67af613562882ee0461d2aa7115-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 06dfc67af613562882ee0461d2aa7115 columnFamilyName cf1 2024-11-12T14:43:35,512 DEBUG [StoreOpener-06dfc67af613562882ee0461d2aa7115-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:43:35,513 INFO [StoreOpener-06dfc67af613562882ee0461d2aa7115-1 {}] regionserver.HStore(327): Store=06dfc67af613562882ee0461d2aa7115/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T14:43:35,513 INFO [StoreOpener-06dfc67af613562882ee0461d2aa7115-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 06dfc67af613562882ee0461d2aa7115 2024-11-12T14:43:35,514 INFO [StoreOpener-06dfc67af613562882ee0461d2aa7115-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 06dfc67af613562882ee0461d2aa7115 columnFamilyName cf2 2024-11-12T14:43:35,514 DEBUG [StoreOpener-06dfc67af613562882ee0461d2aa7115-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:43:35,515 INFO [StoreOpener-06dfc67af613562882ee0461d2aa7115-1 {}] regionserver.HStore(327): Store=06dfc67af613562882ee0461d2aa7115/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T14:43:35,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41431 {}] regionserver.HRegion(1722): Closing 06dfc67af613562882ee0461d2aa7115, disabling compactions & flushes 2024-11-12T14:43:35,515 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41431 {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1731422614808.06dfc67af613562882ee0461d2aa7115. 2024-11-12T14:43:35,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41431 {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731422614808.06dfc67af613562882ee0461d2aa7115. 2024-11-12T14:43:35,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41431 {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731422614808.06dfc67af613562882ee0461d2aa7115. after waiting 0 ms 2024-11-12T14:43:35,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41431 {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1731422614808.06dfc67af613562882ee0461d2aa7115. 2024-11-12T14:43:35,516 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41431 {}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1731422614808.06dfc67af613562882ee0461d2aa7115. 2024-11-12T14:43:35,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41431 {}] regionserver.HRegion(1676): Region close journal for 06dfc67af613562882ee0461d2aa7115: Waiting for close lock at 1731422615515Disabling compacts and flushes for region at 1731422615515Disabling writes for close at 1731422615515Writing region close event to WAL at 1731422615516 (+1 ms)Closed at 1731422615516 2024-11-12T14:43:35,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34233 {}] procedure.ProcedureSyncWait(219): waitFor pid=7 2024-11-12T14:43:35,670 INFO [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(122): Close 06dfc67af613562882ee0461d2aa7115 2024-11-12T14:43:35,670 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-12T14:43:35,670 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1722): Closing 06dfc67af613562882ee0461d2aa7115, disabling compactions & flushes 2024-11-12T14:43:35,670 INFO [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1731422614808.06dfc67af613562882ee0461d2aa7115. 2024-11-12T14:43:35,671 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731422614808.06dfc67af613562882ee0461d2aa7115. 2024-11-12T14:43:35,671 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731422614808.06dfc67af613562882ee0461d2aa7115. after waiting 0 ms 2024-11-12T14:43:35,671 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1731422614808.06dfc67af613562882ee0461d2aa7115. 2024-11-12T14:43:35,671 INFO [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(2902): Flushing 06dfc67af613562882ee0461d2aa7115 2/2 column families, dataSize=31 B heapSize=616 B 2024-11-12T14:43:35,693 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40559/user/jenkins/test-data/41a388f8-1764-825e-e285-25964d37c9f3/data/default/testReplayEditsAfterRegionMovedWithMultiCF/06dfc67af613562882ee0461d2aa7115/.tmp/cf1/d3973c6c3aef45048c9f51516d7ae384 is 35, key is r1/cf1:q/1731422615486/Put/seqid=0 2024-11-12T14:43:35,699 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36175 is added to blk_1073741887_1065 (size=4783) 2024-11-12T14:43:35,699 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37395 is added to blk_1073741887_1065 (size=4783) 2024-11-12T14:43:35,699 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45975 is added to blk_1073741887_1065 (size=4783) 2024-11-12T14:43:35,700 INFO [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31 B at sequenceid=5 (bloomFilter=false), to=hdfs://localhost:40559/user/jenkins/test-data/41a388f8-1764-825e-e285-25964d37c9f3/data/default/testReplayEditsAfterRegionMovedWithMultiCF/06dfc67af613562882ee0461d2aa7115/.tmp/cf1/d3973c6c3aef45048c9f51516d7ae384 2024-11-12T14:43:35,707 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40559/user/jenkins/test-data/41a388f8-1764-825e-e285-25964d37c9f3/data/default/testReplayEditsAfterRegionMovedWithMultiCF/06dfc67af613562882ee0461d2aa7115/.tmp/cf1/d3973c6c3aef45048c9f51516d7ae384 as hdfs://localhost:40559/user/jenkins/test-data/41a388f8-1764-825e-e285-25964d37c9f3/data/default/testReplayEditsAfterRegionMovedWithMultiCF/06dfc67af613562882ee0461d2aa7115/cf1/d3973c6c3aef45048c9f51516d7ae384 2024-11-12T14:43:35,713 INFO [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40559/user/jenkins/test-data/41a388f8-1764-825e-e285-25964d37c9f3/data/default/testReplayEditsAfterRegionMovedWithMultiCF/06dfc67af613562882ee0461d2aa7115/cf1/d3973c6c3aef45048c9f51516d7ae384, entries=1, sequenceid=5, filesize=4.7 K 2024-11-12T14:43:35,714 INFO [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~31 B/31, heapSize ~344 B/344, currentSize=0 B/0 for 06dfc67af613562882ee0461d2aa7115 in 43ms, sequenceid=5, compaction requested=false 2024-11-12T14:43:35,715 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testReplayEditsAfterRegionMovedWithMultiCF' 2024-11-12T14:43:35,720 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40559/user/jenkins/test-data/41a388f8-1764-825e-e285-25964d37c9f3/data/default/testReplayEditsAfterRegionMovedWithMultiCF/06dfc67af613562882ee0461d2aa7115/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-12T14:43:35,722 INFO [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1731422614808.06dfc67af613562882ee0461d2aa7115. 2024-11-12T14:43:35,723 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1676): Region close journal for 06dfc67af613562882ee0461d2aa7115: Waiting for close lock at 1731422615670Running coprocessor pre-close hooks at 1731422615670Disabling compacts and flushes for region at 1731422615670Disabling writes for close at 1731422615671 (+1 ms)Obtaining lock to block concurrent updates at 1731422615671Preparing flush snapshotting stores in 06dfc67af613562882ee0461d2aa7115 at 1731422615671Finished memstore snapshotting testReplayEditsAfterRegionMovedWithMultiCF,,1731422614808.06dfc67af613562882ee0461d2aa7115., syncing WAL and waiting on mvcc, flushsize=dataSize=31, getHeapSize=584, getOffHeapSize=0, getCellsCount=1 at 1731422615671Flushing stores of testReplayEditsAfterRegionMovedWithMultiCF,,1731422614808.06dfc67af613562882ee0461d2aa7115. at 1731422615672 (+1 ms)Flushing 06dfc67af613562882ee0461d2aa7115/cf1: creating writer at 1731422615673 (+1 ms)Flushing 06dfc67af613562882ee0461d2aa7115/cf1: appending metadata at 1731422615693 (+20 ms)Flushing 06dfc67af613562882ee0461d2aa7115/cf1: closing flushed file at 1731422615693Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@425eec45: reopening flushed file at 1731422615706 (+13 ms)Finished flush of dataSize ~31 B/31, heapSize ~344 B/344, currentSize=0 B/0 for 06dfc67af613562882ee0461d2aa7115 in 43ms, sequenceid=5, compaction requested=false at 1731422615715 (+9 ms)Writing region close event to WAL at 1731422615716 (+1 ms)Running coprocessor post-close hooks at 1731422615721 (+5 ms)Closed at 1731422615722 (+1 ms) 2024-11-12T14:43:35,723 INFO [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionServer(3302): Adding 06dfc67af613562882ee0461d2aa7115 move to 3945e1ca6f47,41431,1731422599613 record at close sequenceid=5 2024-11-12T14:43:35,726 INFO [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(157): Closed 06dfc67af613562882ee0461d2aa7115 2024-11-12T14:43:35,727 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=06dfc67af613562882ee0461d2aa7115, regionState=CLOSED 2024-11-12T14:43:35,730 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure 06dfc67af613562882ee0461d2aa7115, server=3945e1ca6f47,38727,1731422599737 because future has completed 2024-11-12T14:43:35,735 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-12T14:43:35,735 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; CloseRegionProcedure 06dfc67af613562882ee0461d2aa7115, server=3945e1ca6f47,38727,1731422599737 in 221 msec 2024-11-12T14:43:35,737 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=06dfc67af613562882ee0461d2aa7115, REOPEN/MOVE; state=CLOSED, location=3945e1ca6f47,41431,1731422599613; forceNewPlan=false, retain=false 2024-11-12T14:43:35,887 INFO [3945e1ca6f47:34233 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-12T14:43:35,888 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=06dfc67af613562882ee0461d2aa7115, regionState=OPENING, regionLocation=3945e1ca6f47,41431,1731422599613 2024-11-12T14:43:35,891 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=06dfc67af613562882ee0461d2aa7115, REOPEN/MOVE because future has completed 2024-11-12T14:43:35,892 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 06dfc67af613562882ee0461d2aa7115, server=3945e1ca6f47,41431,1731422599613}] 2024-11-12T14:43:36,054 INFO [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1731422614808.06dfc67af613562882ee0461d2aa7115. 2024-11-12T14:43:36,054 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => 06dfc67af613562882ee0461d2aa7115, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1731422614808.06dfc67af613562882ee0461d2aa7115.', STARTKEY => '', ENDKEY => ''} 2024-11-12T14:43:36,055 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF 06dfc67af613562882ee0461d2aa7115 2024-11-12T14:43:36,055 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1731422614808.06dfc67af613562882ee0461d2aa7115.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T14:43:36,055 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for 06dfc67af613562882ee0461d2aa7115 2024-11-12T14:43:36,055 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for 06dfc67af613562882ee0461d2aa7115 2024-11-12T14:43:36,057 INFO [StoreOpener-06dfc67af613562882ee0461d2aa7115-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 06dfc67af613562882ee0461d2aa7115 2024-11-12T14:43:36,058 INFO [StoreOpener-06dfc67af613562882ee0461d2aa7115-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 06dfc67af613562882ee0461d2aa7115 columnFamilyName cf1 2024-11-12T14:43:36,058 DEBUG [StoreOpener-06dfc67af613562882ee0461d2aa7115-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:43:36,064 DEBUG [StoreOpener-06dfc67af613562882ee0461d2aa7115-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:40559/user/jenkins/test-data/41a388f8-1764-825e-e285-25964d37c9f3/data/default/testReplayEditsAfterRegionMovedWithMultiCF/06dfc67af613562882ee0461d2aa7115/cf1/d3973c6c3aef45048c9f51516d7ae384 2024-11-12T14:43:36,064 INFO [StoreOpener-06dfc67af613562882ee0461d2aa7115-1 {}] regionserver.HStore(327): Store=06dfc67af613562882ee0461d2aa7115/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T14:43:36,064 INFO [StoreOpener-06dfc67af613562882ee0461d2aa7115-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 06dfc67af613562882ee0461d2aa7115 2024-11-12T14:43:36,065 INFO [StoreOpener-06dfc67af613562882ee0461d2aa7115-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 06dfc67af613562882ee0461d2aa7115 columnFamilyName cf2 2024-11-12T14:43:36,065 DEBUG [StoreOpener-06dfc67af613562882ee0461d2aa7115-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:43:36,066 INFO [StoreOpener-06dfc67af613562882ee0461d2aa7115-1 {}] regionserver.HStore(327): Store=06dfc67af613562882ee0461d2aa7115/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T14:43:36,066 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for 06dfc67af613562882ee0461d2aa7115 2024-11-12T14:43:36,066 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40559/user/jenkins/test-data/41a388f8-1764-825e-e285-25964d37c9f3/data/default/testReplayEditsAfterRegionMovedWithMultiCF/06dfc67af613562882ee0461d2aa7115 2024-11-12T14:43:36,068 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40559/user/jenkins/test-data/41a388f8-1764-825e-e285-25964d37c9f3/data/default/testReplayEditsAfterRegionMovedWithMultiCF/06dfc67af613562882ee0461d2aa7115 2024-11-12T14:43:36,069 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for 06dfc67af613562882ee0461d2aa7115 2024-11-12T14:43:36,069 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for 06dfc67af613562882ee0461d2aa7115 2024-11-12T14:43:36,069 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-11-12T14:43:36,071 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for 06dfc67af613562882ee0461d2aa7115 2024-11-12T14:43:36,072 INFO [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1114): Opened 06dfc67af613562882ee0461d2aa7115; next sequenceid=9; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63788403, jitterRate=-0.04947872459888458}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-11-12T14:43:36,073 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 06dfc67af613562882ee0461d2aa7115 2024-11-12T14:43:36,073 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for 06dfc67af613562882ee0461d2aa7115: Running coprocessor pre-open hook at 1731422616055Writing region info on filesystem at 1731422616055Initializing all the Stores at 1731422616056 (+1 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731422616056Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731422616056Cleaning up temporary data from old regions at 1731422616069 (+13 ms)Running coprocessor post-open hooks at 1731422616073 (+4 ms)Region opened successfully at 1731422616073 2024-11-12T14:43:36,075 INFO [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1731422614808.06dfc67af613562882ee0461d2aa7115., pid=9, masterSystemTime=1731422616048 2024-11-12T14:43:36,077 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1731422614808.06dfc67af613562882ee0461d2aa7115. 2024-11-12T14:43:36,077 INFO [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1731422614808.06dfc67af613562882ee0461d2aa7115. 2024-11-12T14:43:36,078 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=06dfc67af613562882ee0461d2aa7115, regionState=OPEN, openSeqNum=9, regionLocation=3945e1ca6f47,41431,1731422599613 2024-11-12T14:43:36,081 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 06dfc67af613562882ee0461d2aa7115, server=3945e1ca6f47,41431,1731422599613 because future has completed 2024-11-12T14:43:36,085 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=7 2024-11-12T14:43:36,085 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=7, state=SUCCESS, hasLock=false; OpenRegionProcedure 06dfc67af613562882ee0461d2aa7115, server=3945e1ca6f47,41431,1731422599613 in 190 msec 2024-11-12T14:43:36,087 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=06dfc67af613562882ee0461d2aa7115, REOPEN/MOVE in 581 msec 2024-11-12T14:43:36,112 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-12T14:43:36,114 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55454, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-12T14:43:36,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38727 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 139 connection: 172.17.0.2:59246 deadline: 1731422676118, exception=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=3945e1ca6f47 port=41431 startCode=1731422599613. As of locationSeqNum=5. 2024-11-12T14:43:36,140 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1731422614808.06dfc67af613562882ee0461d2aa7115., hostname=3945e1ca6f47,38727,1731422599737, seqNum=2 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1731422614808.06dfc67af613562882ee0461d2aa7115., hostname=3945e1ca6f47,38727,1731422599737, seqNum=2, error=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=3945e1ca6f47 port=41431 startCode=1731422599613. As of locationSeqNum=5. 2024-11-12T14:43:36,141 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1731422614808.06dfc67af613562882ee0461d2aa7115., hostname=3945e1ca6f47,38727,1731422599737, seqNum=2 is org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=3945e1ca6f47 port=41431 startCode=1731422599613. As of locationSeqNum=5. 2024-11-12T14:43:36,141 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(84): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1731422614808.06dfc67af613562882ee0461d2aa7115., hostname=3945e1ca6f47,38727,1731422599737, seqNum=2 with the new location region=testReplayEditsAfterRegionMovedWithMultiCF,,1731422614808.06dfc67af613562882ee0461d2aa7115., hostname=3945e1ca6f47,41431,1731422599613, seqNum=5 constructed by org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=3945e1ca6f47 port=41431 startCode=1731422599613. As of locationSeqNum=5. 2024-11-12T14:43:36,259 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T14:43:36,261 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55456, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T14:43:36,273 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 06dfc67af613562882ee0461d2aa7115 2/2 column families, dataSize=50 B heapSize=720 B 2024-11-12T14:43:36,298 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40559/user/jenkins/test-data/41a388f8-1764-825e-e285-25964d37c9f3/data/default/testReplayEditsAfterRegionMovedWithMultiCF/06dfc67af613562882ee0461d2aa7115/.tmp/cf1/2ae4a17743964ee495b896da021d354b is 29, key is r1/cf1:/1731422616263/DeleteFamily/seqid=0 2024-11-12T14:43:36,311 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36175 is added to blk_1073741888_1066 (size=4906) 2024-11-12T14:43:36,315 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37395 is added to blk_1073741888_1066 (size=4906) 2024-11-12T14:43:36,315 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=25 B at sequenceid=12 (bloomFilter=false), to=hdfs://localhost:40559/user/jenkins/test-data/41a388f8-1764-825e-e285-25964d37c9f3/data/default/testReplayEditsAfterRegionMovedWithMultiCF/06dfc67af613562882ee0461d2aa7115/.tmp/cf1/2ae4a17743964ee495b896da021d354b 2024-11-12T14:43:36,315 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45975 is added to blk_1073741888_1066 (size=4906) 2024-11-12T14:43:36,324 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 2ae4a17743964ee495b896da021d354b 2024-11-12T14:43:36,344 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40559/user/jenkins/test-data/41a388f8-1764-825e-e285-25964d37c9f3/data/default/testReplayEditsAfterRegionMovedWithMultiCF/06dfc67af613562882ee0461d2aa7115/.tmp/cf2/1b2583b5076e46d5863d77d74c1e1419 is 29, key is r1/cf2:/1731422616263/DeleteFamily/seqid=0 2024-11-12T14:43:36,361 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45975 is added to blk_1073741889_1067 (size=4906) 2024-11-12T14:43:36,361 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37395 is added to blk_1073741889_1067 (size=4906) 2024-11-12T14:43:36,361 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36175 is added to blk_1073741889_1067 (size=4906) 2024-11-12T14:43:36,364 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=25 B at sequenceid=12 (bloomFilter=false), to=hdfs://localhost:40559/user/jenkins/test-data/41a388f8-1764-825e-e285-25964d37c9f3/data/default/testReplayEditsAfterRegionMovedWithMultiCF/06dfc67af613562882ee0461d2aa7115/.tmp/cf2/1b2583b5076e46d5863d77d74c1e1419 2024-11-12T14:43:36,371 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 1b2583b5076e46d5863d77d74c1e1419 2024-11-12T14:43:36,372 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40559/user/jenkins/test-data/41a388f8-1764-825e-e285-25964d37c9f3/data/default/testReplayEditsAfterRegionMovedWithMultiCF/06dfc67af613562882ee0461d2aa7115/.tmp/cf1/2ae4a17743964ee495b896da021d354b as hdfs://localhost:40559/user/jenkins/test-data/41a388f8-1764-825e-e285-25964d37c9f3/data/default/testReplayEditsAfterRegionMovedWithMultiCF/06dfc67af613562882ee0461d2aa7115/cf1/2ae4a17743964ee495b896da021d354b 2024-11-12T14:43:36,381 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 2ae4a17743964ee495b896da021d354b 2024-11-12T14:43:36,381 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40559/user/jenkins/test-data/41a388f8-1764-825e-e285-25964d37c9f3/data/default/testReplayEditsAfterRegionMovedWithMultiCF/06dfc67af613562882ee0461d2aa7115/cf1/2ae4a17743964ee495b896da021d354b, entries=1, sequenceid=12, filesize=4.8 K 2024-11-12T14:43:36,382 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40559/user/jenkins/test-data/41a388f8-1764-825e-e285-25964d37c9f3/data/default/testReplayEditsAfterRegionMovedWithMultiCF/06dfc67af613562882ee0461d2aa7115/.tmp/cf2/1b2583b5076e46d5863d77d74c1e1419 as hdfs://localhost:40559/user/jenkins/test-data/41a388f8-1764-825e-e285-25964d37c9f3/data/default/testReplayEditsAfterRegionMovedWithMultiCF/06dfc67af613562882ee0461d2aa7115/cf2/1b2583b5076e46d5863d77d74c1e1419 2024-11-12T14:43:36,391 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 1b2583b5076e46d5863d77d74c1e1419 2024-11-12T14:43:36,391 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40559/user/jenkins/test-data/41a388f8-1764-825e-e285-25964d37c9f3/data/default/testReplayEditsAfterRegionMovedWithMultiCF/06dfc67af613562882ee0461d2aa7115/cf2/1b2583b5076e46d5863d77d74c1e1419, entries=1, sequenceid=12, filesize=4.8 K 2024-11-12T14:43:36,392 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~50 B/50, heapSize ~688 B/688, currentSize=0 B/0 for 06dfc67af613562882ee0461d2aa7115 in 119ms, sequenceid=12, compaction requested=false 2024-11-12T14:43:36,393 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 06dfc67af613562882ee0461d2aa7115: 2024-11-12T14:43:36,396 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-12T14:43:36,398 DEBUG [Time-limited test {}] regionserver.HStore(1541): 06dfc67af613562882ee0461d2aa7115/cf1 is initiating major compaction (all files) 2024-11-12T14:43:36,398 INFO [Time-limited test {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-12T14:43:36,398 INFO [Time-limited test {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-12T14:43:36,398 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of 06dfc67af613562882ee0461d2aa7115/cf1 in testReplayEditsAfterRegionMovedWithMultiCF,,1731422614808.06dfc67af613562882ee0461d2aa7115. 2024-11-12T14:43:36,399 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40559/user/jenkins/test-data/41a388f8-1764-825e-e285-25964d37c9f3/data/default/testReplayEditsAfterRegionMovedWithMultiCF/06dfc67af613562882ee0461d2aa7115/cf1/d3973c6c3aef45048c9f51516d7ae384, hdfs://localhost:40559/user/jenkins/test-data/41a388f8-1764-825e-e285-25964d37c9f3/data/default/testReplayEditsAfterRegionMovedWithMultiCF/06dfc67af613562882ee0461d2aa7115/cf1/2ae4a17743964ee495b896da021d354b] into tmpdir=hdfs://localhost:40559/user/jenkins/test-data/41a388f8-1764-825e-e285-25964d37c9f3/data/default/testReplayEditsAfterRegionMovedWithMultiCF/06dfc67af613562882ee0461d2aa7115/.tmp, totalSize=9.5 K 2024-11-12T14:43:36,400 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting d3973c6c3aef45048c9f51516d7ae384, keycount=1, bloomtype=NONE, size=4.7 K, encoding=NONE, compression=NONE, seqNum=5, earliestPutTs=1731422615486 2024-11-12T14:43:36,401 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 2ae4a17743964ee495b896da021d354b, keycount=1, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=9223372036854775807 2024-11-12T14:43:36,415 INFO [Time-limited test {}] throttle.PressureAwareThroughputController(145): 06dfc67af613562882ee0461d2aa7115#cf1#compaction#16 average throughput is NaN MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T14:43:36,422 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36175 is added to blk_1073741890_1068 (size=4626) 2024-11-12T14:43:36,422 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45975 is added to blk_1073741890_1068 (size=4626) 2024-11-12T14:43:36,423 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37395 is added to blk_1073741890_1068 (size=4626) 2024-11-12T14:43:36,441 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40559/user/jenkins/test-data/41a388f8-1764-825e-e285-25964d37c9f3/data/default/testReplayEditsAfterRegionMovedWithMultiCF/06dfc67af613562882ee0461d2aa7115/.tmp/cf1/44a277ae66ab433ba4efd1edc131dfb4 as hdfs://localhost:40559/user/jenkins/test-data/41a388f8-1764-825e-e285-25964d37c9f3/data/default/testReplayEditsAfterRegionMovedWithMultiCF/06dfc67af613562882ee0461d2aa7115/cf1/44a277ae66ab433ba4efd1edc131dfb4 2024-11-12T14:43:36,456 INFO [Time-limited test {}] regionserver.HStore(1337): Completed major compaction of 2 (all) file(s) in 06dfc67af613562882ee0461d2aa7115/cf1 of 06dfc67af613562882ee0461d2aa7115 into 44a277ae66ab433ba4efd1edc131dfb4(size=4.5 K), total size for store is 4.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T14:43:36,456 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for 06dfc67af613562882ee0461d2aa7115: 2024-11-12T14:43:36,457 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 1 store files, 0 compacting, 1 eligible, 16 blocking 2024-11-12T14:43:36,457 DEBUG [Time-limited test {}] regionserver.HStore(1541): 06dfc67af613562882ee0461d2aa7115/cf2 is initiating major compaction (all files) 2024-11-12T14:43:36,457 INFO [Time-limited test {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-12T14:43:36,457 INFO [Time-limited test {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-12T14:43:36,457 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of 06dfc67af613562882ee0461d2aa7115/cf2 in testReplayEditsAfterRegionMovedWithMultiCF,,1731422614808.06dfc67af613562882ee0461d2aa7115. 2024-11-12T14:43:36,457 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40559/user/jenkins/test-data/41a388f8-1764-825e-e285-25964d37c9f3/data/default/testReplayEditsAfterRegionMovedWithMultiCF/06dfc67af613562882ee0461d2aa7115/cf2/1b2583b5076e46d5863d77d74c1e1419] into tmpdir=hdfs://localhost:40559/user/jenkins/test-data/41a388f8-1764-825e-e285-25964d37c9f3/data/default/testReplayEditsAfterRegionMovedWithMultiCF/06dfc67af613562882ee0461d2aa7115/.tmp, totalSize=4.8 K 2024-11-12T14:43:36,458 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 1b2583b5076e46d5863d77d74c1e1419, keycount=1, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=9223372036854775807 2024-11-12T14:43:36,464 INFO [Time-limited test {}] throttle.PressureAwareThroughputController(145): 06dfc67af613562882ee0461d2aa7115#cf2#compaction#17 average throughput is NaN MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T14:43:36,473 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45975 is added to blk_1073741891_1069 (size=4592) 2024-11-12T14:43:36,473 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37395 is added to blk_1073741891_1069 (size=4592) 2024-11-12T14:43:36,474 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36175 is added to blk_1073741891_1069 (size=4592) 2024-11-12T14:43:36,482 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40559/user/jenkins/test-data/41a388f8-1764-825e-e285-25964d37c9f3/data/default/testReplayEditsAfterRegionMovedWithMultiCF/06dfc67af613562882ee0461d2aa7115/.tmp/cf2/49854fb177e64856a73947a38e838905 as hdfs://localhost:40559/user/jenkins/test-data/41a388f8-1764-825e-e285-25964d37c9f3/data/default/testReplayEditsAfterRegionMovedWithMultiCF/06dfc67af613562882ee0461d2aa7115/cf2/49854fb177e64856a73947a38e838905 2024-11-12T14:43:36,489 INFO [Time-limited test {}] regionserver.HStore(1337): Completed major compaction of 1 (all) file(s) in 06dfc67af613562882ee0461d2aa7115/cf2 of 06dfc67af613562882ee0461d2aa7115 into 49854fb177e64856a73947a38e838905(size=4.5 K), total size for store is 4.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T14:43:36,489 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for 06dfc67af613562882ee0461d2aa7115: 2024-11-12T14:43:36,493 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34233 {}] master.HMaster(2410): Client=jenkins//172.17.0.2 move hri=06dfc67af613562882ee0461d2aa7115, source=3945e1ca6f47,41431,1731422599613, destination=3945e1ca6f47,38727,1731422599737, warming up region on 3945e1ca6f47,38727,1731422599737 2024-11-12T14:43:36,493 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34233 {}] master.HMaster(2414): Client=jenkins//172.17.0.2 move hri=06dfc67af613562882ee0461d2aa7115, source=3945e1ca6f47,41431,1731422599613, destination=3945e1ca6f47,38727,1731422599737, running balancer 2024-11-12T14:43:36,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34233 {}] procedure2.ProcedureExecutor(1139): Stored pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=06dfc67af613562882ee0461d2aa7115, REOPEN/MOVE 2024-11-12T14:43:36,495 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=06dfc67af613562882ee0461d2aa7115, REOPEN/MOVE 2024-11-12T14:43:36,496 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=06dfc67af613562882ee0461d2aa7115, regionState=CLOSING, regionLocation=3945e1ca6f47,41431,1731422599613 2024-11-12T14:43:36,497 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38727 {}] regionserver.RSRpcServices(2066): Warmup testReplayEditsAfterRegionMovedWithMultiCF,,1731422614808.06dfc67af613562882ee0461d2aa7115. 2024-11-12T14:43:36,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38727 {}] regionserver.HRegion(7855): Warmup {ENCODED => 06dfc67af613562882ee0461d2aa7115, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1731422614808.06dfc67af613562882ee0461d2aa7115.', STARTKEY => '', ENDKEY => ''} 2024-11-12T14:43:36,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38727 {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1731422614808.06dfc67af613562882ee0461d2aa7115.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T14:43:36,498 INFO [StoreOpener-06dfc67af613562882ee0461d2aa7115-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 06dfc67af613562882ee0461d2aa7115 2024-11-12T14:43:36,499 INFO [StoreOpener-06dfc67af613562882ee0461d2aa7115-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 06dfc67af613562882ee0461d2aa7115 columnFamilyName cf1 2024-11-12T14:43:36,499 DEBUG [StoreOpener-06dfc67af613562882ee0461d2aa7115-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:43:36,499 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=06dfc67af613562882ee0461d2aa7115, REOPEN/MOVE because future has completed 2024-11-12T14:43:36,500 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-12T14:43:36,500 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=11, ppid=10, state=RUNNABLE, hasLock=false; CloseRegionProcedure 06dfc67af613562882ee0461d2aa7115, server=3945e1ca6f47,41431,1731422599613}] 2024-11-12T14:43:36,508 INFO [StoreFileOpener-06dfc67af613562882ee0461d2aa7115-cf1-1 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 2ae4a17743964ee495b896da021d354b 2024-11-12T14:43:36,508 DEBUG [StoreOpener-06dfc67af613562882ee0461d2aa7115-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:40559/user/jenkins/test-data/41a388f8-1764-825e-e285-25964d37c9f3/data/default/testReplayEditsAfterRegionMovedWithMultiCF/06dfc67af613562882ee0461d2aa7115/cf1/2ae4a17743964ee495b896da021d354b 2024-11-12T14:43:36,514 DEBUG [StoreOpener-06dfc67af613562882ee0461d2aa7115-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:40559/user/jenkins/test-data/41a388f8-1764-825e-e285-25964d37c9f3/data/default/testReplayEditsAfterRegionMovedWithMultiCF/06dfc67af613562882ee0461d2aa7115/cf1/44a277ae66ab433ba4efd1edc131dfb4 2024-11-12T14:43:36,523 DEBUG [StoreOpener-06dfc67af613562882ee0461d2aa7115-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:40559/user/jenkins/test-data/41a388f8-1764-825e-e285-25964d37c9f3/data/default/testReplayEditsAfterRegionMovedWithMultiCF/06dfc67af613562882ee0461d2aa7115/cf1/d3973c6c3aef45048c9f51516d7ae384 2024-11-12T14:43:36,523 INFO [StoreOpener-06dfc67af613562882ee0461d2aa7115-1 {}] regionserver.HStore(327): Store=06dfc67af613562882ee0461d2aa7115/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T14:43:36,523 INFO [StoreOpener-06dfc67af613562882ee0461d2aa7115-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 06dfc67af613562882ee0461d2aa7115 2024-11-12T14:43:36,524 INFO [StoreOpener-06dfc67af613562882ee0461d2aa7115-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 06dfc67af613562882ee0461d2aa7115 columnFamilyName cf2 2024-11-12T14:43:36,524 DEBUG [StoreOpener-06dfc67af613562882ee0461d2aa7115-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:43:36,534 INFO [StoreFileOpener-06dfc67af613562882ee0461d2aa7115-cf2-1 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 1b2583b5076e46d5863d77d74c1e1419 2024-11-12T14:43:36,534 DEBUG [StoreOpener-06dfc67af613562882ee0461d2aa7115-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:40559/user/jenkins/test-data/41a388f8-1764-825e-e285-25964d37c9f3/data/default/testReplayEditsAfterRegionMovedWithMultiCF/06dfc67af613562882ee0461d2aa7115/cf2/1b2583b5076e46d5863d77d74c1e1419 2024-11-12T14:43:36,543 DEBUG [StoreOpener-06dfc67af613562882ee0461d2aa7115-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:40559/user/jenkins/test-data/41a388f8-1764-825e-e285-25964d37c9f3/data/default/testReplayEditsAfterRegionMovedWithMultiCF/06dfc67af613562882ee0461d2aa7115/cf2/49854fb177e64856a73947a38e838905 2024-11-12T14:43:36,543 INFO [StoreOpener-06dfc67af613562882ee0461d2aa7115-1 {}] regionserver.HStore(327): Store=06dfc67af613562882ee0461d2aa7115/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T14:43:36,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38727 {}] regionserver.HRegion(1722): Closing 06dfc67af613562882ee0461d2aa7115, disabling compactions & flushes 2024-11-12T14:43:36,544 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38727 {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1731422614808.06dfc67af613562882ee0461d2aa7115. 2024-11-12T14:43:36,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38727 {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731422614808.06dfc67af613562882ee0461d2aa7115. 2024-11-12T14:43:36,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38727 {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731422614808.06dfc67af613562882ee0461d2aa7115. after waiting 0 ms 2024-11-12T14:43:36,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38727 {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1731422614808.06dfc67af613562882ee0461d2aa7115. 2024-11-12T14:43:36,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34233 {}] procedure.ProcedureSyncWait(219): waitFor pid=10 2024-11-12T14:43:36,554 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38727 {}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1731422614808.06dfc67af613562882ee0461d2aa7115. 2024-11-12T14:43:36,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38727 {}] regionserver.HRegion(1676): Region close journal for 06dfc67af613562882ee0461d2aa7115: Waiting for close lock at 1731422616544Disabling compacts and flushes for region at 1731422616544Disabling writes for close at 1731422616544Writing region close event to WAL at 1731422616554 (+10 ms)Closed at 1731422616554 2024-11-12T14:43:36,653 INFO [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] handler.UnassignRegionHandler(122): Close 06dfc67af613562882ee0461d2aa7115 2024-11-12T14:43:36,653 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-12T14:43:36,653 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1722): Closing 06dfc67af613562882ee0461d2aa7115, disabling compactions & flushes 2024-11-12T14:43:36,653 INFO [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1731422614808.06dfc67af613562882ee0461d2aa7115. 2024-11-12T14:43:36,653 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731422614808.06dfc67af613562882ee0461d2aa7115. 2024-11-12T14:43:36,653 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731422614808.06dfc67af613562882ee0461d2aa7115. after waiting 0 ms 2024-11-12T14:43:36,653 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1731422614808.06dfc67af613562882ee0461d2aa7115. 2024-11-12T14:43:36,654 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1731422614808.06dfc67af613562882ee0461d2aa7115.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40559/user/jenkins/test-data/41a388f8-1764-825e-e285-25964d37c9f3/data/default/testReplayEditsAfterRegionMovedWithMultiCF/06dfc67af613562882ee0461d2aa7115/cf1/d3973c6c3aef45048c9f51516d7ae384, hdfs://localhost:40559/user/jenkins/test-data/41a388f8-1764-825e-e285-25964d37c9f3/data/default/testReplayEditsAfterRegionMovedWithMultiCF/06dfc67af613562882ee0461d2aa7115/cf1/2ae4a17743964ee495b896da021d354b] to archive 2024-11-12T14:43:36,657 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1731422614808.06dfc67af613562882ee0461d2aa7115.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-12T14:43:36,661 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1731422614808.06dfc67af613562882ee0461d2aa7115.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40559/user/jenkins/test-data/41a388f8-1764-825e-e285-25964d37c9f3/data/default/testReplayEditsAfterRegionMovedWithMultiCF/06dfc67af613562882ee0461d2aa7115/cf1/d3973c6c3aef45048c9f51516d7ae384 to hdfs://localhost:40559/user/jenkins/test-data/41a388f8-1764-825e-e285-25964d37c9f3/archive/data/default/testReplayEditsAfterRegionMovedWithMultiCF/06dfc67af613562882ee0461d2aa7115/cf1/d3973c6c3aef45048c9f51516d7ae384 2024-11-12T14:43:36,662 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1731422614808.06dfc67af613562882ee0461d2aa7115.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40559/user/jenkins/test-data/41a388f8-1764-825e-e285-25964d37c9f3/data/default/testReplayEditsAfterRegionMovedWithMultiCF/06dfc67af613562882ee0461d2aa7115/cf1/2ae4a17743964ee495b896da021d354b to hdfs://localhost:40559/user/jenkins/test-data/41a388f8-1764-825e-e285-25964d37c9f3/archive/data/default/testReplayEditsAfterRegionMovedWithMultiCF/06dfc67af613562882ee0461d2aa7115/cf1/2ae4a17743964ee495b896da021d354b 2024-11-12T14:43:36,677 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1731422614808.06dfc67af613562882ee0461d2aa7115.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40559/user/jenkins/test-data/41a388f8-1764-825e-e285-25964d37c9f3/data/default/testReplayEditsAfterRegionMovedWithMultiCF/06dfc67af613562882ee0461d2aa7115/cf2/1b2583b5076e46d5863d77d74c1e1419] to archive 2024-11-12T14:43:36,678 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1731422614808.06dfc67af613562882ee0461d2aa7115.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-12T14:43:36,680 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1731422614808.06dfc67af613562882ee0461d2aa7115.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40559/user/jenkins/test-data/41a388f8-1764-825e-e285-25964d37c9f3/data/default/testReplayEditsAfterRegionMovedWithMultiCF/06dfc67af613562882ee0461d2aa7115/cf2/1b2583b5076e46d5863d77d74c1e1419 to hdfs://localhost:40559/user/jenkins/test-data/41a388f8-1764-825e-e285-25964d37c9f3/archive/data/default/testReplayEditsAfterRegionMovedWithMultiCF/06dfc67af613562882ee0461d2aa7115/cf2/1b2583b5076e46d5863d77d74c1e1419 2024-11-12T14:43:36,711 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40559/user/jenkins/test-data/41a388f8-1764-825e-e285-25964d37c9f3/data/default/testReplayEditsAfterRegionMovedWithMultiCF/06dfc67af613562882ee0461d2aa7115/recovered.edits/17.seqid, newMaxSeqId=17, maxSeqId=8 2024-11-12T14:43:36,712 INFO [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1731422614808.06dfc67af613562882ee0461d2aa7115. 2024-11-12T14:43:36,712 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1676): Region close journal for 06dfc67af613562882ee0461d2aa7115: Waiting for close lock at 1731422616653Running coprocessor pre-close hooks at 1731422616653Disabling compacts and flushes for region at 1731422616653Disabling writes for close at 1731422616653Writing region close event to WAL at 1731422616682 (+29 ms)Running coprocessor post-close hooks at 1731422616712 (+30 ms)Closed at 1731422616712 2024-11-12T14:43:36,712 INFO [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegionServer(3302): Adding 06dfc67af613562882ee0461d2aa7115 move to 3945e1ca6f47,38727,1731422599737 record at close sequenceid=12 2024-11-12T14:43:36,716 INFO [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] handler.UnassignRegionHandler(157): Closed 06dfc67af613562882ee0461d2aa7115 2024-11-12T14:43:36,718 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=06dfc67af613562882ee0461d2aa7115, regionState=CLOSED 2024-11-12T14:43:36,723 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=10, state=RUNNABLE, hasLock=false; CloseRegionProcedure 06dfc67af613562882ee0461d2aa7115, server=3945e1ca6f47,41431,1731422599613 because future has completed 2024-11-12T14:43:36,738 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=11, resume processing ppid=10 2024-11-12T14:43:36,738 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=10, state=SUCCESS, hasLock=false; CloseRegionProcedure 06dfc67af613562882ee0461d2aa7115, server=3945e1ca6f47,41431,1731422599613 in 232 msec 2024-11-12T14:43:36,739 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=06dfc67af613562882ee0461d2aa7115, REOPEN/MOVE; state=CLOSED, location=3945e1ca6f47,38727,1731422599737; forceNewPlan=false, retain=false 2024-11-12T14:43:36,890 INFO [3945e1ca6f47:34233 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-12T14:43:36,890 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=06dfc67af613562882ee0461d2aa7115, regionState=OPENING, regionLocation=3945e1ca6f47,38727,1731422599737 2024-11-12T14:43:36,893 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=06dfc67af613562882ee0461d2aa7115, REOPEN/MOVE because future has completed 2024-11-12T14:43:36,893 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 06dfc67af613562882ee0461d2aa7115, server=3945e1ca6f47,38727,1731422599737}] 2024-11-12T14:43:37,051 INFO [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1731422614808.06dfc67af613562882ee0461d2aa7115. 2024-11-12T14:43:37,051 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7752): Opening region: {ENCODED => 06dfc67af613562882ee0461d2aa7115, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1731422614808.06dfc67af613562882ee0461d2aa7115.', STARTKEY => '', ENDKEY => ''} 2024-11-12T14:43:37,051 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF 06dfc67af613562882ee0461d2aa7115 2024-11-12T14:43:37,051 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1731422614808.06dfc67af613562882ee0461d2aa7115.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T14:43:37,051 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7794): checking encryption for 06dfc67af613562882ee0461d2aa7115 2024-11-12T14:43:37,052 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7797): checking classloading for 06dfc67af613562882ee0461d2aa7115 2024-11-12T14:43:37,053 INFO [StoreOpener-06dfc67af613562882ee0461d2aa7115-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 06dfc67af613562882ee0461d2aa7115 2024-11-12T14:43:37,054 INFO [StoreOpener-06dfc67af613562882ee0461d2aa7115-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 06dfc67af613562882ee0461d2aa7115 columnFamilyName cf1 2024-11-12T14:43:37,054 DEBUG [StoreOpener-06dfc67af613562882ee0461d2aa7115-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:43:37,064 DEBUG [StoreOpener-06dfc67af613562882ee0461d2aa7115-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:40559/user/jenkins/test-data/41a388f8-1764-825e-e285-25964d37c9f3/data/default/testReplayEditsAfterRegionMovedWithMultiCF/06dfc67af613562882ee0461d2aa7115/cf1/44a277ae66ab433ba4efd1edc131dfb4 2024-11-12T14:43:37,064 INFO [StoreOpener-06dfc67af613562882ee0461d2aa7115-1 {}] regionserver.HStore(327): Store=06dfc67af613562882ee0461d2aa7115/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T14:43:37,064 INFO [StoreOpener-06dfc67af613562882ee0461d2aa7115-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 06dfc67af613562882ee0461d2aa7115 2024-11-12T14:43:37,066 INFO [StoreOpener-06dfc67af613562882ee0461d2aa7115-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 06dfc67af613562882ee0461d2aa7115 columnFamilyName cf2 2024-11-12T14:43:37,066 DEBUG [StoreOpener-06dfc67af613562882ee0461d2aa7115-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:43:37,072 DEBUG [StoreOpener-06dfc67af613562882ee0461d2aa7115-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:40559/user/jenkins/test-data/41a388f8-1764-825e-e285-25964d37c9f3/data/default/testReplayEditsAfterRegionMovedWithMultiCF/06dfc67af613562882ee0461d2aa7115/cf2/49854fb177e64856a73947a38e838905 2024-11-12T14:43:37,072 INFO [StoreOpener-06dfc67af613562882ee0461d2aa7115-1 {}] regionserver.HStore(327): Store=06dfc67af613562882ee0461d2aa7115/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T14:43:37,073 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1038): replaying wal for 06dfc67af613562882ee0461d2aa7115 2024-11-12T14:43:37,074 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40559/user/jenkins/test-data/41a388f8-1764-825e-e285-25964d37c9f3/data/default/testReplayEditsAfterRegionMovedWithMultiCF/06dfc67af613562882ee0461d2aa7115 2024-11-12T14:43:37,075 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40559/user/jenkins/test-data/41a388f8-1764-825e-e285-25964d37c9f3/data/default/testReplayEditsAfterRegionMovedWithMultiCF/06dfc67af613562882ee0461d2aa7115 2024-11-12T14:43:37,076 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1048): stopping wal replay for 06dfc67af613562882ee0461d2aa7115 2024-11-12T14:43:37,076 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1060): Cleaning up temporary data for 06dfc67af613562882ee0461d2aa7115 2024-11-12T14:43:37,076 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-11-12T14:43:37,078 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1093): writing seq id for 06dfc67af613562882ee0461d2aa7115 2024-11-12T14:43:37,079 INFO [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1114): Opened 06dfc67af613562882ee0461d2aa7115; next sequenceid=18; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65233216, jitterRate=-0.02794933319091797}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-11-12T14:43:37,079 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 06dfc67af613562882ee0461d2aa7115 2024-11-12T14:43:37,080 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1006): Region open journal for 06dfc67af613562882ee0461d2aa7115: Running coprocessor pre-open hook at 1731422617052Writing region info on filesystem at 1731422617052Initializing all the Stores at 1731422617053 (+1 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731422617053Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731422617053Cleaning up temporary data from old regions at 1731422617076 (+23 ms)Running coprocessor post-open hooks at 1731422617079 (+3 ms)Region opened successfully at 1731422617080 (+1 ms) 2024-11-12T14:43:37,081 INFO [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1731422614808.06dfc67af613562882ee0461d2aa7115., pid=12, masterSystemTime=1731422617046 2024-11-12T14:43:37,083 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1731422614808.06dfc67af613562882ee0461d2aa7115. 2024-11-12T14:43:37,083 INFO [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1731422614808.06dfc67af613562882ee0461d2aa7115. 2024-11-12T14:43:37,084 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=06dfc67af613562882ee0461d2aa7115, regionState=OPEN, openSeqNum=18, regionLocation=3945e1ca6f47,38727,1731422599737 2024-11-12T14:43:37,087 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=12, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 06dfc67af613562882ee0461d2aa7115, server=3945e1ca6f47,38727,1731422599737 because future has completed 2024-11-12T14:43:37,091 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=10 2024-11-12T14:43:37,091 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=10, state=SUCCESS, hasLock=false; OpenRegionProcedure 06dfc67af613562882ee0461d2aa7115, server=3945e1ca6f47,38727,1731422599737 in 195 msec 2024-11-12T14:43:37,093 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=06dfc67af613562882ee0461d2aa7115, REOPEN/MOVE in 598 msec 2024-11-12T14:43:37,097 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-12T14:43:37,098 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59260, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-12T14:43:37,100 ERROR [Time-limited test {}] regionserver.HRegionServer(2442): ***** ABORTING region server 3945e1ca6f47,38727,1731422599737: testing ***** 2024-11-12T14:43:37,100 ERROR [Time-limited test {}] regionserver.HRegionServer(2447): RegionServer abort: loaded coprocessors are: [org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint] 2024-11-12T14:43:37,102 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for java.lang:type=Memory 2024-11-12T14:43:37,103 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=IPC 2024-11-12T14:43:37,105 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Replication 2024-11-12T14:43:37,107 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Server 2024-11-12T14:43:37,124 INFO [Time-limited test {}] regionserver.HRegionServer(2451): Dump of metrics as JSON on abort: { "beans": [ { "name": "java.lang:type=Memory", "modelerType": "sun.management.MemoryImpl", "ObjectPendingFinalizationCount": 0, "HeapMemoryUsage": { "committed": 1048576000, "init": 1048576000, "max": 2306867200, "used": 478629736 }, "NonHeapMemoryUsage": { "committed": 171769856, "init": 7667712, "max": -1, "used": 169113104 }, "Verbose": false, "ObjectName": "java.lang:type=Memory" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=IPC", "modelerType": "RegionServer,sub=IPC", "tag.Context": "regionserver", "tag.Hostname": "3945e1ca6f47", "queueSize": 0, "numCallsInGeneralQueue": 0, "numCallsInReplicationQueue": 0, "numCallsInBulkLoadQueue": 0, "numCallsInPriorityQueue": 0, "numCallsInMetaPriorityQueue": 0, "numOpenConnections": 0, "numActiveHandler": 0, "numActiveGeneralHandler": 0, "numActivePriorityHandler": 0, "numActiveReplicationHandler": 0, "numGeneralCallsDropped": 0, "numLifoModeSwitches": 0, "numCallsInWriteQueue": 0, "numActiveBulkLoadHandler": 0, "numCallsInReadQueue": 0, "numCallsInScanQueue": 0, "numActiveWriteHandler": 0, "numActiveReadHandler": 0, "numActiveScanHandler": 0, "nettyDirectMemoryUsage": 67108864, "nettyTotalPendingOutboundBytes": 0, "nettyMaxPendingOutboundBytes": 0, "receivedBytes": 2068, "exceptions.RegionMovedException": 0, "authenticationSuccesses": 0, "authorizationFailures": 0, "exceptions.requestTooBig": 0, "UnwritableTime_num_ops": 0, "UnwritableTime_min": 0, "UnwritableTime_max": 0, "UnwritableTime_mean": 0, "UnwritableTime_25th_percentile": 0, "UnwritableTime_median": 0, "UnwritableTime_75th_percentile": 0, "UnwritableTime_90th_percentile": 0, "UnwritableTime_95th_percentile": 0, "UnwritableTime_98th_percentile": 0, "UnwritableTime_99th_percentile": 0, "UnwritableTime_99.9th_percentile": 0, "exceptions.OutOfOrderScannerNextException": 0, "exceptions.rpcThrottling": 0, "exceptions.otherExceptions": 0, "ProcessCallTime_num_ops": 8, "ProcessCallTime_min": 1, "ProcessCallTime_max": 9, "ProcessCallTime_mean": 4, "ProcessCallTime_25th_percentile": 3, "ProcessCallTime_median": 5, "ProcessCallTime_75th_percentile": 7, "ProcessCallTime_90th_percentile": 8, "ProcessCallTime_95th_percentile": 8, "ProcessCallTime_98th_percentile": 8, "ProcessCallTime_99th_percentile": 8, "ProcessCallTime_99.9th_percentile": 8, "ProcessCallTime_TimeRangeCount_0-1": 8, "exceptions.callQueueTooBig": 0, "QueueCallTime_num_ops": 8, "QueueCallTime_min": 0, "QueueCallTime_max": 1, "QueueCallTime_mean": 0, "QueueCallTime_25th_percentile": 0, "QueueCallTime_median": 0, "QueueCallTime_75th_percentile": 0, "QueueCallTime_90th_percentile": 0, "QueueCallTime_95th_percentile": 0, "QueueCallTime_98th_percentile": 0, "QueueCallTime_99th_percentile": 0, "QueueCallTime_99.9th_percentile": 0, "QueueCallTime_TimeRangeCount_0-1": 8, "authenticationFailures": 0, "exceptions.multiResponseTooLarge": 0, "exceptions.callDropped": 0, "TotalCallTime_num_ops": 8, "TotalCallTime_min": 1, "TotalCallTime_max": 9, "TotalCallTime_mean": 4, "TotalCallTime_25th_percentile": 3, "TotalCallTime_median": 5, "TotalCallTime_75th_percentile": 7, "TotalCallTime_90th_percentile": 8, "TotalCallTime_95th_percentile": 8, "TotalCallTime_98th_percentile": 8, "TotalCallTime_99th_percentile": 8, "TotalCallTime_99.9th_percentile": 8, "TotalCallTime_TimeRangeCount_0-1": 8, "exceptions.RegionTooBusyException": 0, "exceptions.FailedSanityCheckException": 0, "ResponseSize_num_ops": 8, "ResponseSize_min": 0, "ResponseSize_max": 175, "ResponseSize_mean": 50, "ResponseSize_25th_percentile": 43, "ResponseSize_median": 87, "ResponseSize_75th_percentile": 131, "ResponseSize_90th_percentile": 157, "ResponseSize_95th_percentile": 166, "ResponseSize_98th_percentile": 171, "ResponseSize_99th_percentile": 173, "ResponseSize_99.9th_percentile": 174, "ResponseSize_SizeRangeCount_0-10": 8, "exceptions.UnknownScannerException": 0, "exceptions": 0, "maxOutboundBytesExceeded": 0, "authenticationFallbacks": 0, "exceptions.quotaExceeded": 0, "exceptions.callTimedOut": 0, "exceptions.NotServingRegionException": 0, "authorizationSuccesses": 0, "exceptions.ScannerResetException": 0, "RequestSize_num_ops": 8, "RequestSize_min": 89, "RequestSize_max": 121, "RequestSize_mean": 103, "RequestSize_25th_percentile": 97, "RequestSize_median": 105, "RequestSize_75th_percentile": 113, "RequestSize_90th_percentile": 117, "RequestSize_95th_percentile": 119, "RequestSize_98th_percentile": 120, "RequestSize_99th_percentile": 120, "RequestSize_99.9th_percentile": 120, "RequestSize_SizeRangeCount_0-10": 8, "sentBytes": 352 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Replication", "modelerType": "RegionServer,sub=Replication", "tag.Context": "regionserver", "tag.Hostname": "3945e1ca6f47", "source.shippedHFiles": 0, "Source.ageOfLastShippedOp_num_ops": 0, "Source.ageOfLastShippedOp_min": 0, "Source.ageOfLastShippedOp_max": 0, "Source.ageOfLastShippedOp_mean": 0, "Source.ageOfLastShippedOp_25th_percentile": 0, "Source.ageOfLastShippedOp_median": 0, "Source.ageOfLastShippedOp_75th_percentile": 0, "Source.ageOfLastShippedOp_90th_percentile": 0, "Source.ageOfLastShippedOp_95th_percentile": 0, "Source.ageOfLastShippedOp_98th_percentile": 0, "Source.ageOfLastShippedOp_99th_percentile": 0, "Source.ageOfLastShippedOp_99.9th_percentile": 0, "source.uncleanlyClosedLogs": 0, "source.closedLogsWithUnknownFileLength": 0, "source.walReaderEditsBufferUsage": 0, "source.repeatedLogFileBytes": 0, "source.sizeOfHFileRefsQueue": 0, "source.logReadInBytes": 0, "source.completedRecoverQueues": 0, "source.sizeOfLogQueue": 0, "source.restartedLogReading": 0, "source.failedRecoverQueues": 0, "source.ignoredUncleanlyClosedLogContentsInBytes": 0, "Sink.ageOfLastAppliedOp_num_ops": 0, "Sink.ageOfLastAppliedOp_min": 0, "Sink.ageOfLastAppliedOp_max": 0, "Sink.ageOfLastAppliedOp_mean": 0, "Sink.ageOfLastAppliedOp_25th_percentile": 0, "Sink.ageOfLastAppliedOp_median": 0, "Sink.ageOfLastAppliedOp_75th_percentile": 0, "Sink.ageOfLastAppliedOp_90th_percentile": 0, "Sink.ageOfLastAppliedOp_95th_percentile": 0, "Sink.ageOfLastAppliedOp_98th_percentile": 0, "Sink.ageOfLastAppliedOp_99th_percentile": 0, "Sink.ageOfLastAppliedOp_99.9th_percentile": 0, "source.logEditsRead": 0, "source.numInitializing": 0, "source.shippedOps": 0, "sink.appliedHFiles": 0, "source.logEditsFiltered": 0, "source.shippedBytes": 0, "sink.appliedOps": 0, "source.completedLogs": 0, "source.failedBatches": 0, "sink.failedBatches": 0, "source.shippedBatches": 0, "sink.appliedBatches": 0 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Server", "modelerType": "RegionServer,sub=Server", "tag.zookeeperQuorum": "127.0.0.1:55341", "tag.serverName": "3945e1ca6f47,33387,1731422599699", "tag.clusterId": "70468ada-4082-4450-9033-a07d7e00c036", "tag.Context": "regionserver", "tag.Hostname": "3945e1ca6f47", "regionCount": 1, "storeCount": 4, "hlogFileCount": 2, "hlogFileSize": 0, "storeFileCount": 0, "maxStoreFileCount": 0, "memStoreSize": 74, "memStoreHeapSize": 1248, "memStoreOffHeapSize": 0, "storeFileSize": 0, "storeFileSizeGrowthRate": 0.0, "maxStoreFileAge": 0, "minStoreFileAge": 0, "avgStoreFileAge": 0, "numReferenceFiles": 0, "regionServerStartTime": 1731422599699, "averageRegionSize": 74, "storeFileIndexSize": 0, "staticIndexSize": 0, "staticBloomSize": 0, "bloomFilterRequestsCount": 0, "bloomFilterNegativeResultsCount": 0, "bloomFilterEligibleRequestsCount": 0, "mutationsWithoutWALCount": 0, "mutationsWithoutWALSize": 0, "percentFilesLocal": 0.0, "percentFilesLocalSecondaryRegions": 0.0, "totalBytesRead": 10034, "localBytesRead": 10034, "shortCircuitBytesRead": 0, "zeroCopyBytesRead": 0, "splitQueueLength": 0, "compactionQueueLength": 0, "smallCompactionQueueLength": 0, "largeCompactionQueueLength": 0, "flushQueueLength": 0, "blockCacheFreeSize": 922070024, "blockCacheCount": 0, "blockCacheDataBlockCount": 0, "blockCacheSize": 676856, "blockCacheCountHitPercent": 0.0, "blockCacheExpressHitPercent": 0.0, "l1CacheSize": 676856, "l1CacheFreeSize": 922070024, "l1CacheCount": 0, "l1CacheEvictionCount": 0, "l1CacheHitCount": 0, "l1CacheMissCount": 0, "l1CacheHitRatio": 0.0, "l1CacheMissRatio": 0.0, "l2CacheSize": 0, "l2CacheFreeSize": 0, "l2CacheCount": 0, "l2CacheEvictionCount": 0, "l2CacheHitCount": 0, "l2CacheMissCount": 0, "l2CacheHitRatio": 0.0, "l2CacheMissRatio": 0.0, "mobFileCacheCount": 0, "mobFileCacheHitPercent": 0.0, "readRequestRatePerSecond": 1.4, "writeRequestRatePerSecond": 0.4, "ByteBuffAllocatorHeapAllocationBytes": 4782, "ByteBuffAllocatorPoolAllocationBytes": 199680, "ByteBuffAllocatorHeapAllocationRatio": 0.0, "ByteBuffAllocatorTotalBufferCount": 186, "ByteBuffAllocatorUsedBufferCount": 1, "activeScanners": 0, "totalRequestCount": 7, "totalRowActionRequestCount": 9, "readRequestCount": 7, "cpRequestCount": 0, "filteredReadRequestCount": 0, "writeRequestCount": 2, "rpcGetRequestCount": 1, "rpcFullScanRequestCount": 0, "rpcScanRequestCount": 4, "rpcMultiRequestCount": 0, "rpcMutateRequestCount": 2, "checkMutateFailedCount": 0, "checkMutatePassedCount": 0, "blockCacheHitCount": 0, "blockCacheHitCountPrimary": 0, "blockCacheHitCachingCount": 0, "blockCacheMissCount": 0, "blockCacheMissCountPrimary": 0, "blockCacheMissCachingCount": 0, "blockCacheEvictionCount": 0, "blockCacheEvictionCountPrimary": 0, "blockCacheFailedInsertionCount": 0, "blockCacheDataMissCount": 0, "blockCacheLeafIndexMissCount": 0, "blockCacheBloomChunkMissCount": 0, "blockCacheMetaMissCount": 0, "blockCacheRootIndexMissCount": 0, "blockCacheIntermediateIndexMissCount": 0, "blockCacheFileInfoMissCount": 0, "blockCacheGeneralBloomMetaMissCount": 0, "blockCacheDeleteFamilyBloomMissCount": 0, "blockCacheTrailerMissCount": 0, "blockCacheDataHitCount": 0, "blockCacheLeafIndexHitCount": 0, "blockCacheBloomChunkHitCount": 0, "blockCacheMetaHitCount": 0, "blockCacheRootIndexHitCount": 0, "blockCacheIntermediateIndexHitCount": 0, "blockCacheFileInfoHitCount": 0, "blockCacheGeneralBloomMetaHitCount": 0, "blockCacheDeleteFamilyBloomHitCount": 0, "blockCacheTrailerHitCount": 0, "updatesBlockedTime": 0, "flushedCellsCount": 0, "compactedCellsCount": 0, "majorCompactedCellsCount": 0, "flushedCellsSize": 0, "compactedCellsSize": 0, "majorCompactedCellsSize": 0, "cellsCountCompactedFromMob": 0, "cellsCountCompactedToMob": 0, "cellsSizeCompactedFromMob": 0, "cellsSizeCompactedToMob": 0, "mobFlushCount": 0, "mobFlushedCellsCount": 0, "mobFlushedCellsSize": 0, "mobScanCellsCount": 0, "mobScanCellsSize": 0, "mobFileCacheAccessCount": 0, "mobFileCacheMissCount": 0, "mobFileCacheEvictedCount": 0, "hedgedReads": 0, "hedgedReadWins": 0, "hedgedReadOpsInCurThread": 0, "blockedRequestCount": 0, "CheckAndMutate_num_ops": 0, "CheckAndMutate_min": 0, "CheckAndMutate_max": 0, "CheckAndMutate_mean": 0, "CheckAndMutate_25th_percentile": 0, "CheckAndMutate_median": 0, "CheckAndMutate_75th_percentile": 0, "CheckAndMutate_90th_percentile": 0, "CheckAndMutate_95th_percentile": 0, "CheckAndMutate_98th_percentile": 0, "CheckAndMutate_99th_percentile": 0, "CheckAndMutate_99.9th_percentile": 0, "MajorCompactionTime_num_ops": 0, "MajorCompactionTime_min": 0, "MajorCompactionTime_max": 0, "MajorCompactionTime_mean": 0, "MajorCompactionTime_25th_percentile": 0, "MajorCompactionTime_median": 0, "MajorCompactionTime_75th_percentile": 0, "MajorCompactionTime_90th_percentile": 0, "MajorCompactionTime_95th_percentile": 0, "MajorCompactionTime_98th_percentile": 0, "MajorCompactionTime_99th_percentile": 0, "MajorCompactionTime_99.9th_percentile": 0, "ScanTime_num_ops": 8, "ScanTime_min": 0, "ScanTime_max": 2, "ScanTime_mean": 1, "ScanTime_25th_percentile": 1, "ScanTime_median": 1, "ScanTime_75th_percentile": 2, "ScanTime_90th_percentile": 2, "ScanTime_95th_percentile": 2, "ScanTime_98th_percentile": 2, "ScanTime_99th_percentile": 2, "ScanTime_99.9th_percentile": 2, "ScanTime_TimeRangeCount_0-1": 3, "CheckAndMutateBlockBytesScanned_num_ops": 0, "CheckAndMutateBlockBytesScanned_min": 0, "CheckAndMutateBlockBytesScanned_max": 0, "CheckAndMutateBlockBytesScanned_mean": 0, "CheckAndMutateBlockBytesScanned_25th_percentile": 0, "CheckAndMutateBlockBytesScanned_median": 0, "CheckAndMutateBlockBytesScanned_75th_percentile": 0, "CheckAndMutateBlockBytesScanned_90th_percentile": 0, "CheckAndMutateBlockBytesScanned_95th_percentile": 0, "CheckAndMutateBlockBytesScanned_98th_percentile": 0, "CheckAndMutateBlockBytesScanned_99th_percentile": 0, "CheckAndMutateBlockBytesScanned_99.9th_percentile": 0, "Put_num_ops": 15, "Put_min": 1, "Put_max": 2, "Put_mean": 1, "Put_25th_percentile": 1, "Put_median": 2, "Put_75th_percentile": 2, "Put_90th_percentile": 2, "Put_95th_percentile": 2, "Put_98th_percentile": 2, "Put_99th_percentile": 2, "Put_99.9th_percentile": 2, "Put_TimeRangeCount_0-1": 4, "splitRequestCount": 0, "AppendBlockBytesScanned_num_ops": 0, "AppendBlockBytesScanned_min": 0, "AppendBlockBytesScanned_max": 0, "AppendBlockBytesScanned_mean": 0, "AppendBlockBytesScanned_25th_percentile": 0, "AppendBlockBytesScanned_median": 0, "AppendBlockBytesScanned_75th_percentile": 0, "AppendBlockBytesScanned_90th_percentile": 0, "AppendBlockBytesScanned_95th_percentile": 0, "AppendBlockBytesScanned_98th_percentile": 0, "AppendBlockBytesScanned_99th_percentile": 0, "AppendBlockBytesScanned_99.9th_percentile": 0, "PutBatch_num_ops": 0, "PutBatch_min": 0, "PutBatch_max": 0, "PutBatch_mean": 0, "PutBatch_25th_percentile": 0, "PutBatch_median": 0, "PutBatch_75th_percentile": 0, "PutBatch_90th_percentile": 0, "PutBatch_95th_percentile": 0, "PutBatch_98th_percentile": 0, "PutBatch_99th_percentile": 0, "PutBatch_99.9th_percentile": 0, "IncrementBlockBytesScanned_num_ops": 0, "IncrementBlockBytesScanned_min": 0, "IncrementBlockBytesScanned_max": 0, "IncrementBlockBytesScanned_mean": 0, "IncrementBlockBytesScanned_25th_percentile": 0, "IncrementBlockBytesScanned_median": 0, "IncrementBlockBytesScanned_75th_percentile": 0, "IncrementBlockBytesScanned_90th_percentile": 0, "IncrementBlockBytesScanned_95th_percentile": 0, "IncrementBlockBytesScanned_98th_percentile": 0, "IncrementBlockBytesScanned_99th_percentile": 0, "IncrementBlockBytesScanned_99.9th_percentile": 0, "SplitTime_num_ops": 0, "SplitTime_min": 0, "SplitTime_max": 0, "SplitTime_mean": 0, "SplitTime_25th_percentile": 0, "SplitTime_median": 0, "SplitTime_75th_percentile": 0, "SplitTime_90th_percentile": 0, "SplitTime_95th_percentile": 0, "SplitTime_98th_percentile": 0, "SplitTime_99th_percentile": 0, "SplitTime_99.9th_percentile": 0, "GetBlockBytesScanned_num_ops": 0, "GetBlockBytesScanned_min": 0, "GetBlockBytesScanned_max": 0, "GetBlockBytesScanned_mean": 0, "GetBlockBytesScanned_25th_percentile": 0, "GetBlockBytesScanned_median": 0, "GetBlockBytesScanned_75th_percentile": 0, "GetBlockBytesScanned_90th_percentile": 0, "GetBlockBytesScanned_95th_percentile": 0, "GetBlockBytesScanned_98th_percentile": 0, "GetBlockBytesScanned_99th_percentile": 0, "GetBlockBytesScanned_99.9th_percentile": 0, "majorCompactedInputBytes": 0, "slowAppendCount": 0, "flushedOutputBytes": 0, "Replay_num_ops": 0, "Replay_min": 0, "Replay_max": 0, "Replay_mean": 0, "Replay_25th_percentile": 0, "Replay_median": 0, "Replay_75th_percentile": 0, "Replay_90th_percentile": 0, "Replay_95th_percentile": 0, "Replay_98th_percentile": 0, "Replay_99th_percentile": 0, "Replay_99.9th_percentile": 0, "MajorCompactionInputSize_num_ops": 0, "MajorCompactionInputSize_min": 0, "MajorCompactionInputSize_max": 0, "MajorCompactionInputSize_mean": 0, "MajorCompactionInputSize_25th_percentile": 0, "MajorCompactionInputSize_median": 0, "MajorCompactionInputSize_75th_percentile": 0, "MajorCompactionInputSize_90th_percentile": 0, "MajorCompactionInputSize_95th_percentile": 0, "MajorCompactionInputSize_98th_percentile": 0, "MajorCompactionInputSize_99th_percentile": 0, "MajorCompactionInputSize_99.9th_percentile": 0, "pauseInfoThresholdExceeded": 0, "CheckAndDelete_num_ops": 0, "CheckAndDelete_min": 0, "CheckAndDelete_max": 0, "CheckAndDelete_mean": 0, "CheckAndDelete_25th_percentile": 0, "CheckAndDelete_median": 0, "CheckAndDelete_75th_percentile": 0, "CheckAndDelete_90th_percentile": 0, "CheckAndDelete_95th_percentile": 0, "CheckAndDelete_98th_percentile": 0, "CheckAndDelete_99th_percentile": 0, "CheckAndDelete_99.9th_percentile": 0, "CompactionInputSize_num_ops": 0, "CompactionInputSize_min": 0, "CompactionInputSize_max": 0, "CompactionInputSize_mean": 0, "CompactionInputSize_25th_percentile": 0, "CompactionInputSize_median": 0, "CompactionInputSize_75th_percentile": 0, "CompactionInputSize_90th_percentile": 0, "CompactionInputSize_95th_percentile": 0, "CompactionInputSize_98th_percentile": 0, "CompactionInputSize_99th_percentile": 0, "CompactionInputSize_99.9th_percentile": 0, "flushedMemstoreBytes": 0, "majorCompactedOutputBytes": 0, "slowPutCount": 0, "compactedInputBytes": 0, "FlushOutputSize_num_ops": 0, "FlushOutputSize_min": 0, "FlushOutputSize_max": 0, "FlushOutputSize_mean": 0, "FlushOutputSize_25th_percentile": 0, "FlushOutputSize_median": 0, "FlushOutputSize_75th_percentile": 0, "FlushOutputSize_90th_percentile": 0, "FlushOutputSize_95th_percentile": 0, "FlushOutputSize_98th_percentile": 0, "FlushOutputSize_99th_percentile": 0, "FlushOutputSize_99.9th_percentile": 0, "PauseTimeWithGc_num_ops": 0, "PauseTimeWithGc_min": 0, "PauseTimeWithGc_max": 0, "PauseTimeWithGc_mean": 0, "PauseTimeWithGc_25th_percentile": 0, "PauseTimeWithGc_median": 0, "PauseTimeWithGc_75th_percentile": 0, "PauseTimeWithGc_90th_percentile": 0, "PauseTimeWithGc_95th_percentile": 0, "PauseTimeWithGc_98th_percentile": 0, "PauseTimeWithGc_99th_percentile": 0, "PauseTimeWithGc_99.9th_percentile": 0, "compactedOutputBytes": 0, "pauseWarnThresholdExceeded": 0, "ScanBlockBytesScanned_num_ops": 0, "ScanBlockBytesScanned_min": 0, "ScanBlockBytesScanned_max": 0, "ScanBlockBytesScanned_mean": 0, "ScanBlockBytesScanned_25th_percentile": 0, "ScanBlockBytesScanned_median": 0, "ScanBlockBytesScanned_75th_percentile": 0, "ScanBlockBytesScanned_90th_percentile": 0, "ScanBlockBytesScanned_95th_percentile": 0, "ScanBlockBytesScanned_98th_percentile": 0, "ScanBlockBytesScanned_99th_percentile": 0, "ScanBlockBytesScanned_99.9th_percentile": 0, "Increment_num_ops": 0, "Increment_min": 0, "Increment_max": 0, "Increment_mean": 0, "Increment_25th_percentile": 0, "Increment_median": 0, "Increment_75th_percentile": 0, "Increment_90th_percentile": 0, "Increment_95th_percentile": 0, "Increment_98th_percentile": 0, "Increment_99th_percentile": 0, "Increment_99.9th_percentile": 0, "Delete_num_ops": 0, "Delete_min": 0, "Delete_max": 0, "Delete_mean": 0, "Delete_25th_percentile": 0, "Delete_median": 0, "Delete_75th_percentile": 0, "Delete_90th_percentile": 0, "Delete_95th_percentile": 0, "Delete_98th_percentile": 0, "Delete_99th_percentile": 0, "Delete_99.9th_percentile": 0, "DeleteBatch_num_ops": 0, "DeleteBatch_min": 0, "DeleteBatch_max": 0, "DeleteBatch_mean": 0, "DeleteBatch_25th_percentile": 0, "DeleteBatch_median": 0, "DeleteBatch_75th_percentile": 0, "DeleteBatch_90th_percentile": 0, "DeleteBatch_95th_percentile": 0, "DeleteBatch_98th_percentile": 0, "DeleteBatch_99th_percentile": 0, "DeleteBatch_99.9th_percentile": 0, "blockBytesScannedCount": 0, "FlushMemstoreSize_num_ops": 0, "FlushMemstoreSize_min": 0, "FlushMemstoreSize_max": 0, "FlushMemstoreSize_mean": 0, "FlushMemstoreSize_25th_percentile": 0, "FlushMemstoreSize_median": 0, "FlushMemstoreSize_75th_percentile": 0, "FlushMemstoreSize_90th_percentile": 0, "FlushMemstoreSize_95th_percentile": 0, "FlushMemstoreSize_98th_percentile": 0, "FlushMemstoreSize_99th_percentile": 0, "FlushMemstoreSize_99.9th_percentile": 0, "CompactionInputFileCount_num_ops": 0, "CompactionInputFileCount_min": 0, "CompactionInputFileCount_max": 0, "CompactionInputFileCount_mean": 0, "CompactionInputFileCount_25th_percentile": 0, "CompactionInputFileCount_median": 0, "CompactionInputFileCount_75th_percentile": 0, "CompactionInputFileCount_90th_percentile": 0, "CompactionInputFileCount_95th_percentile": 0, "CompactionInputFileCount_98th_percentile": 0, "CompactionInputFileCount_99th_percentile": 0, "CompactionInputFileCount_99.9th_percentile": 0, "CompactionTime_num_ops": 0, "CompactionTime_min": 0, "CompactionTime_max": 0, "CompactionTime_mean": 0, "CompactionTime_25th_percentile": 0, "CompactionTime_median": 0, "CompactionTime_75th_percentile": 0, "CompactionTime_90th_percentile": 0, "CompactionTime_95th_percentile": 0, "CompactionTime_98th_percentile": 0, "CompactionTime_99th_percentile": 0, "CompactionTime_99.9th_percentile": 0, "Get_num_ops": 2, "Get_min": 1, "Get_max": 1, "Get_mean": 1, "Get_25th_percentile": 1, "Get_median": 1, "Get_75th_percentile": 1, "Get_90th_percentile": 1, "Get_95th_percentile": 1, "Get_98th_percentile": 1, "Get_99th_percentile": 1, "Get_99.9th_percentile": 1, "Get_TimeRangeCount_0-1": 1, "MajorCompactionInputFileCount_num_ops": 0, "MajorCompactionInputFileCount_min": 0, "MajorCompactionInputFileCount_max": 0, "MajorCompactionInputFileCount_mean": 0, "MajorCompactionInputFileCount_25th_percentile": 0, "MajorCompactionInputFileCount_median": 0, "MajorCompactionInputFileCount_75th_percentile": 0, "MajorCompactionInputFileCount_90th_percentile": 0, "MajorCompactionInputFileCount_95th_percentile": 0, "MajorCompactionInputFileCount_98th_percentile": 0, "MajorCompactionInputFileCount_99th_percentile": 0, "MajorCompactionInputFileCount_99.9th_percentile": 0, "scannerLeaseExpiredCount": 0, "CheckAndPut_num_ops": 0, "CheckAndPut_min": 0, "CheckAndPut_max": 0, "CheckAndPut_mean": 0, "CheckAndPut_25th_percentile": 0, "CheckAndPut_median": 0, "CheckAndPut_75th_percentile": 0, "CheckAndPut_90th_percentile": 0, "CheckAndPut_95th_percentile": 0, "CheckAndPut_98th_percentile": 0, "CheckAndPut_99th_percentile": 0, "CheckAndPut_99.9th_percentile": 0, "MajorCompactionOutputSize_num_ops": 0, "MajorCompactionOutputSize_min": 0, "MajorCompactionOutputSize_max": 0, "MajorCompactionOutputSize_mean": 0, "MajorCompactionOutputSize_25th_percentile": 0, "MajorCompactionOutputSize_median": 0, "MajorCompactionOutputSize_75th_percentile": 0, "MajorCompactionOutputSize_90th_percentile": 0, "MajorCompactionOutputSize_95th_percentile": 0, "MajorCompactionOutputSize_98th_percentile": 0, "MajorCompactionOutputSize_99th_percentile": 0, "MajorCompactionOutputSize_99.9th_percentile": 0, "CompactionOutputFileCount_num_ops": 0, "CompactionOutputFileCount_min": 0, "CompactionOutputFileCount_max": 0, "CompactionOutputFileCount_mean": 0, "CompactionOutputFileCount_25th_percentile": 0, "CompactionOutputFileCount_median": 0, "CompactionOutputFileCount_75th_percentile": 0, "CompactionOutputFileCount_90th_percentile": 0, "CompactionOutputFileCount_95th_percentile": 0, "CompactionOutputFileCount_98th_percentile": 0, "CompactionOutputFileCount_99th_percentile": 0, "CompactionOutputFileCount_99.9th_percentile": 0, "slowDeleteCount": 0, "FlushTime_num_ops": 0, "FlushTime_min": 0, "FlushTime_max": 0, "FlushTime_mean": 0, "FlushTime_25th_percentile": 0, "FlushTime_median": 0, "FlushTime_75th_percentile": 0, "FlushTime_90th_percentile": 0, "FlushTime_95th_percentile": 0, "FlushTime_98th_percentile": 0, "FlushTime_99th_percentile": 0, "FlushTime_99.9th_percentile": 0, "splitSuccessCount": 0, "MajorCompactionOutputFileCount_num_ops": 0, "MajorCompactionOutputFileCount_min": 0, "MajorCompactionOutputFileCount_max": 0, "MajorCompactionOutputFileCount_mean": 0, "MajorCompactionOutputFileCount_25th_percentile": 0, "MajorCompactionOutputFileCount_median": 0, "MajorCompactionOutputFileCount_75th_percentile": 0, "MajorCompactionOutputFileCount_90th_percentile": 0, "MajorCompactionOutputFileCount_95th_percentile": 0, "MajorCompactionOutputFileCount_98th_percentile": 0, "MajorCompactionOutputFileCount_99th_percentile": 0, "MajorCompactionOutputFileCount_99.9th_percentile": 0, "slowGetCount": 0, "ScanSize_num_ops": 8, "ScanSize_min": 1112, "ScanSize_max": 1112, "ScanSize_mean": 1112, "ScanSize_25th_percentile": 1112, "ScanSize_median": 1112, "ScanSize_75th_percentile": 1112, "ScanSize_90th_percentile": 1112, "ScanSize_95th_percentile": 1112, "ScanSize_98th_percentile": 1112, "ScanSize_99th_percentile": 1112, "ScanSize_99.9th_percentile": 1112, "ScanSize_SizeRangeCount_0-10": 4, "CompactionOutputSize_num_ops": 0, "CompactionOutputSize_min": 0, "CompactionOutputSize_max": 0, "CompactionOutputSize_mean": 0, "CompactionOutputSize_25th_percentile": 0, "CompactionOutputSize_median": 0, "CompactionOutputSize_75th_percentile": 0, "CompactionOutputSize_90th_percentile": 0, "CompactionOutputSize_95th_percentile": 0, "CompactionOutputSize_98th_percentile": 0, "CompactionOutputSize_99th_percentile": 0, "CompactionOutputSize_99.9th_percentile": 0, "PauseTimeWithoutGc_num_ops": 0, "PauseTimeWithoutGc_min": 0, "PauseTimeWithoutGc_max": 0, "PauseTimeWithoutGc_mean": 0, "PauseTimeWithoutGc_25th_percentile": 0, "PauseTimeWithoutGc_median": 0, "PauseTimeWithoutGc_75th_percentile": 0, "PauseTimeWithoutGc_90th_percentile": 0, "PauseTimeWithoutGc_95th_percentile": 0, "PauseTimeWithoutGc_98th_percentile": 0, "PauseTimeWithoutGc_99th_percentile": 0, "PauseTimeWithoutGc_99.9th_percentile": 0, "slowIncrementCount": 0, "Append_num_ops": 0, "Append_min": 0, "Append_max": 0, "Append_mean": 0, "Append_25th_percentile": 0, "Append_median": 0, "Append_75th_percentile": 0, "Append_90th_percentile": 0, "Append_95th_percentile": 0, "Append_98th_percentile": 0, "Append_99th_percentile": 0, "Append_99.9th_percentile": 0, "Bulkload_count": 0, "Bulkload_mean_rate": 0.0, "Bulkload_1min_rate": 0.0, "Bulkload_5min_rate": 0.0, "Bulkload_15min_rate": 0.0, "Bulkload_num_ops": 0, "Bulkload_min": 0, "Bulkload_max": 0, "Bulkload_mean": 0, "Bulkload_25th_percentile": 0, "Bulkload_median": 0, "Bulkload_75th_percentile": 0, "Bulkload_90th_percentile": 0, "Bulkload_95th_percentile": 0, "Bulkload_98th_percentile": 0, "Bulkload_99th_percentile": 0, "Bulkload_99.9th_percentile": 0 } ] } 2024-11-12T14:43:37,127 WARN [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34233 {}] master.MasterRpcServices(700): 3945e1ca6f47,38727,1731422599737 reported a fatal error: ***** ABORTING region server 3945e1ca6f47,38727,1731422599737: testing ***** 2024-11-12T14:43:37,129 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '3945e1ca6f47,38727,1731422599737' ***** 2024-11-12T14:43:37,129 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: testing 2024-11-12T14:43:37,129 INFO [RS:2;3945e1ca6f47:38727 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-12T14:43:37,130 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-12T14:43:37,130 INFO [RS:2;3945e1ca6f47:38727 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager abruptly. 2024-11-12T14:43:37,130 INFO [RS:2;3945e1ca6f47:38727 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager abruptly. 2024-11-12T14:43:37,130 INFO [RS:2;3945e1ca6f47:38727 {}] regionserver.HRegionServer(3091): Received CLOSE for 06dfc67af613562882ee0461d2aa7115 2024-11-12T14:43:37,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41431 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Get size: 140 connection: 172.17.0.2:55456 deadline: 1731422677130, exception=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=3945e1ca6f47 port=38727 startCode=1731422599737. As of locationSeqNum=12. 2024-11-12T14:43:37,131 INFO [RS:2;3945e1ca6f47:38727 {}] regionserver.HRegionServer(956): aborting server 3945e1ca6f47,38727,1731422599737 2024-11-12T14:43:37,131 INFO [RS:2;3945e1ca6f47:38727 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-12T14:43:37,131 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 06dfc67af613562882ee0461d2aa7115, disabling compactions & flushes 2024-11-12T14:43:37,131 INFO [RS:2;3945e1ca6f47:38727 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:2;3945e1ca6f47:38727. 2024-11-12T14:43:37,131 INFO [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1731422614808.06dfc67af613562882ee0461d2aa7115. 2024-11-12T14:43:37,131 DEBUG [RPCClient-NioEventLoopGroup-6-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1731422614808.06dfc67af613562882ee0461d2aa7115., hostname=3945e1ca6f47,41431,1731422599613, seqNum=5 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1731422614808.06dfc67af613562882ee0461d2aa7115., hostname=3945e1ca6f47,41431,1731422599613, seqNum=5, error=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=3945e1ca6f47 port=38727 startCode=1731422599737. As of locationSeqNum=12. 2024-11-12T14:43:37,132 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731422614808.06dfc67af613562882ee0461d2aa7115. 2024-11-12T14:43:37,132 DEBUG [RS:2;3945e1ca6f47:38727 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-12T14:43:37,132 DEBUG [RPCClient-NioEventLoopGroup-6-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1731422614808.06dfc67af613562882ee0461d2aa7115., hostname=3945e1ca6f47,41431,1731422599613, seqNum=5 is org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=3945e1ca6f47 port=38727 startCode=1731422599737. As of locationSeqNum=12. 2024-11-12T14:43:37,132 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731422614808.06dfc67af613562882ee0461d2aa7115. after waiting 0 ms 2024-11-12T14:43:37,132 DEBUG [RPCClient-NioEventLoopGroup-6-7 {}] client.AsyncRegionLocatorHelper(84): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1731422614808.06dfc67af613562882ee0461d2aa7115., hostname=3945e1ca6f47,41431,1731422599613, seqNum=5 with the new location region=testReplayEditsAfterRegionMovedWithMultiCF,,1731422614808.06dfc67af613562882ee0461d2aa7115., hostname=3945e1ca6f47,38727,1731422599737, seqNum=12 constructed by org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=3945e1ca6f47 port=38727 startCode=1731422599737. As of locationSeqNum=12. 2024-11-12T14:43:37,132 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1731422614808.06dfc67af613562882ee0461d2aa7115. 2024-11-12T14:43:37,132 DEBUG [RS:2;3945e1ca6f47:38727 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T14:43:37,132 INFO [RS:2;3945e1ca6f47:38727 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-12T14:43:37,132 DEBUG [RS:2;3945e1ca6f47:38727 {}] regionserver.HRegionServer(1325): Online Regions={06dfc67af613562882ee0461d2aa7115=testReplayEditsAfterRegionMovedWithMultiCF,,1731422614808.06dfc67af613562882ee0461d2aa7115.} 2024-11-12T14:43:37,133 DEBUG [RS:2;3945e1ca6f47:38727 {}] regionserver.HRegionServer(1351): Waiting on 06dfc67af613562882ee0461d2aa7115 2024-11-12T14:43:37,134 INFO [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1731422614808.06dfc67af613562882ee0461d2aa7115. 2024-11-12T14:43:37,134 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 06dfc67af613562882ee0461d2aa7115: Waiting for close lock at 1731422617131Running coprocessor pre-close hooks at 1731422617131Disabling compacts and flushes for region at 1731422617131Disabling writes for close at 1731422617132 (+1 ms)Writing region close event to WAL at 1731422617133 (+1 ms)Running coprocessor post-close hooks at 1731422617133Closed at 1731422617134 (+1 ms) 2024-11-12T14:43:37,134 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1731422614808.06dfc67af613562882ee0461d2aa7115. 2024-11-12T14:43:37,206 INFO [regionserver/3945e1ca6f47:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-12T14:43:37,243 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server 3945e1ca6f47,38727,1731422599737 aborting at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processRequest(ServerRpcConnection.java:564) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processOneRpc(ServerRpcConnection.java:364) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyServerRpcConnection.process(NettyServerRpcConnection.java:89) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:56) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:31) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:43:37,244 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1731422614808.06dfc67af613562882ee0461d2aa7115., hostname=3945e1ca6f47,38727,1731422599737, seqNum=12 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1731422614808.06dfc67af613562882ee0461d2aa7115., hostname=3945e1ca6f47,38727,1731422599737, seqNum=12, error=org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server 3945e1ca6f47,38727,1731422599737 aborting 2024-11-12T14:43:37,244 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1731422614808.06dfc67af613562882ee0461d2aa7115., hostname=3945e1ca6f47,38727,1731422599737, seqNum=12 is org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server 3945e1ca6f47,38727,1731422599737 aborting 2024-11-12T14:43:37,245 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testReplayEditsAfterRegionMovedWithMultiCF,,1731422614808.06dfc67af613562882ee0461d2aa7115., hostname=3945e1ca6f47,38727,1731422599737, seqNum=12 from cache 2024-11-12T14:43:37,333 INFO [RS:2;3945e1ca6f47:38727 {}] regionserver.HRegionServer(976): stopping server 3945e1ca6f47,38727,1731422599737; all regions closed. 2024-11-12T14:43:37,336 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37395 is added to blk_1073741835_1011 (size=2288) 2024-11-12T14:43:37,336 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45975 is added to blk_1073741835_1011 (size=2288) 2024-11-12T14:43:37,336 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36175 is added to blk_1073741835_1011 (size=2288) 2024-11-12T14:43:37,337 DEBUG [RS:2;3945e1ca6f47:38727 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T14:43:37,337 INFO [RS:2;3945e1ca6f47:38727 {}] regionserver.LeaseManager(133): Closed leases 2024-11-12T14:43:37,337 INFO [RS:2;3945e1ca6f47:38727 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-12T14:43:37,338 INFO [RS:2;3945e1ca6f47:38727 {}] hbase.ChoreService(370): Chore service for: regionserver/3945e1ca6f47:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-12T14:43:37,338 INFO [regionserver/3945e1ca6f47:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-12T14:43:37,338 INFO [RS:2;3945e1ca6f47:38727 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-12T14:43:37,338 INFO [RS:2;3945e1ca6f47:38727 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-12T14:43:37,338 INFO [RS:2;3945e1ca6f47:38727 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-12T14:43:37,338 INFO [RS:2;3945e1ca6f47:38727 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-12T14:43:37,338 INFO [RS:2;3945e1ca6f47:38727 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:38727 2024-11-12T14:43:37,344 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34233-0x100346390eb0000, quorum=127.0.0.1:55341, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-12T14:43:37,344 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38727-0x100346390eb0003, quorum=127.0.0.1:55341, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/3945e1ca6f47,38727,1731422599737 2024-11-12T14:43:37,344 INFO [RS:2;3945e1ca6f47:38727 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-12T14:43:37,345 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [3945e1ca6f47,38727,1731422599737] 2024-11-12T14:43:37,346 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/3945e1ca6f47,38727,1731422599737 already deleted, retry=false 2024-11-12T14:43:37,346 INFO [RegionServerTracker-0 {}] master.ServerManager(695): Processing expiration of 3945e1ca6f47,38727,1731422599737 on 3945e1ca6f47,34233,1731422599017 2024-11-12T14:43:37,351 DEBUG [RegionServerTracker-0 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure 3945e1ca6f47,38727,1731422599737, splitWal=true, meta=false 2024-11-12T14:43:37,354 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(169): Start pid=13, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure 3945e1ca6f47,38727,1731422599737, splitWal=true, meta=false 2024-11-12T14:43:37,354 INFO [RegionServerTracker-0 {}] assignment.AssignmentManager(1999): Scheduled ServerCrashProcedure pid=13 for 3945e1ca6f47,38727,1731422599737 (carryingMeta=false) 3945e1ca6f47,38727,1731422599737/CRASHED/regionCount=1/lock=java.util.concurrent.locks.ReentrantReadWriteLock@7a82f338[Write locks = 1, Read locks = 0], oldState=ONLINE. 2024-11-12T14:43:37,355 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(207): 3945e1ca6f47,38727,1731422599737 had 1 regions 2024-11-12T14:43:37,356 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=13, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure 3945e1ca6f47,38727,1731422599737, splitWal=true, meta=false, isMeta: false 2024-11-12T14:43:37,358 DEBUG [PEWorker-1 {}] master.MasterWalManager(329): Renamed region directory: hdfs://localhost:40559/user/jenkins/test-data/41a388f8-1764-825e-e285-25964d37c9f3/WALs/3945e1ca6f47,38727,1731422599737-splitting 2024-11-12T14:43:37,359 INFO [PEWorker-1 {}] master.SplitWALManager(105): 3945e1ca6f47,38727,1731422599737 WAL count=1, meta=false 2024-11-12T14:43:37,362 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure 3945e1ca6f47%2C38727%2C1731422599737.1731422601050}] 2024-11-12T14:43:37,369 DEBUG [PEWorker-2 {}] master.SplitWALManager(158): Acquired split WAL worker=3945e1ca6f47,33387,1731422599699 2024-11-12T14:43:37,373 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=15, ppid=14, state=RUNNABLE, hasLock=false; SplitWALRemoteProcedure 3945e1ca6f47%2C38727%2C1731422599737.1731422601050, worker=3945e1ca6f47,33387,1731422599699}] 2024-11-12T14:43:37,446 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38727-0x100346390eb0003, quorum=127.0.0.1:55341, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-12T14:43:37,447 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38727-0x100346390eb0003, quorum=127.0.0.1:55341, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-12T14:43:37,447 INFO [RS:2;3945e1ca6f47:38727 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-12T14:43:37,447 INFO [RS:2;3945e1ca6f47:38727 {}] regionserver.HRegionServer(1031): Exiting; stopping=3945e1ca6f47,38727,1731422599737; zookeeper connection closed. 2024-11-12T14:43:37,447 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@58ba6b66 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@58ba6b66 2024-11-12T14:43:37,456 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1731422614808.06dfc67af613562882ee0461d2aa7115., hostname=3945e1ca6f47,38727,1731422599737, seqNum=18] 2024-11-12T14:43:37,458 WARN [RPCClient-NioEventLoopGroup-6-5 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 3945e1ca6f47:38727 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: 3945e1ca6f47/172.17.0.2:38727 Caused by: java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hbase.thirdparty.io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:336) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:339) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:43:37,459 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1731422614808.06dfc67af613562882ee0461d2aa7115., hostname=3945e1ca6f47,38727,1731422599737, seqNum=18 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1731422614808.06dfc67af613562882ee0461d2aa7115., hostname=3945e1ca6f47,38727,1731422599737, seqNum=18, error=java.net.ConnectException: Call to address=3945e1ca6f47:38727 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: 3945e1ca6f47/172.17.0.2:38727 2024-11-12T14:43:37,459 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1731422614808.06dfc67af613562882ee0461d2aa7115., hostname=3945e1ca6f47,38727,1731422599737, seqNum=18 is java.net.ConnectException: Connection refused 2024-11-12T14:43:37,459 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testReplayEditsAfterRegionMovedWithMultiCF,,1731422614808.06dfc67af613562882ee0461d2aa7115., hostname=3945e1ca6f47,38727,1731422599737, seqNum=18 from cache 2024-11-12T14:43:37,460 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] ipc.FailedServers(52): Added failed server with address 3945e1ca6f47:38727 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: 3945e1ca6f47/172.17.0.2:38727 2024-11-12T14:43:37,534 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SplitWALCallable, pid=15 2024-11-12T14:43:37,558 INFO [RS_LOG_REPLAY_OPS-regionserver/3945e1ca6f47:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(299): Splitting hdfs://localhost:40559/user/jenkins/test-data/41a388f8-1764-825e-e285-25964d37c9f3/WALs/3945e1ca6f47,38727,1731422599737-splitting/3945e1ca6f47%2C38727%2C1731422599737.1731422601050, size=2.2 K (2288bytes) 2024-11-12T14:43:37,558 INFO [RS_LOG_REPLAY_OPS-regionserver/3945e1ca6f47:0-0 {event_type=RS_LOG_REPLAY, pid=15}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:40559/user/jenkins/test-data/41a388f8-1764-825e-e285-25964d37c9f3/WALs/3945e1ca6f47,38727,1731422599737-splitting/3945e1ca6f47%2C38727%2C1731422599737.1731422601050 2024-11-12T14:43:37,559 INFO [RS_LOG_REPLAY_OPS-regionserver/3945e1ca6f47:0-0 {event_type=RS_LOG_REPLAY, pid=15}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:40559/user/jenkins/test-data/41a388f8-1764-825e-e285-25964d37c9f3/WALs/3945e1ca6f47,38727,1731422599737-splitting/3945e1ca6f47%2C38727%2C1731422599737.1731422601050 after 0ms 2024-11-12T14:43:37,598 DEBUG [RS_LOG_REPLAY_OPS-regionserver/3945e1ca6f47:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-12T14:43:37,598 INFO [RS_LOG_REPLAY_OPS-regionserver/3945e1ca6f47:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(310): Open hdfs://localhost:40559/user/jenkins/test-data/41a388f8-1764-825e-e285-25964d37c9f3/WALs/3945e1ca6f47,38727,1731422599737-splitting/3945e1ca6f47%2C38727%2C1731422599737.1731422601050 took 40ms 2024-11-12T14:43:37,607 DEBUG [RS_LOG_REPLAY_OPS-regionserver/3945e1ca6f47:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(352): Last flushed sequenceid for 06dfc67af613562882ee0461d2aa7115: last_flushed_sequence_id: 12 store_sequence_id { family_name: "cf1" sequence_id: 12 } store_sequence_id { family_name: "cf2" sequence_id: 12 } 2024-11-12T14:43:37,608 DEBUG [RS_LOG_REPLAY_OPS-regionserver/3945e1ca6f47:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:40559/user/jenkins/test-data/41a388f8-1764-825e-e285-25964d37c9f3/WALs/3945e1ca6f47,38727,1731422599737-splitting/3945e1ca6f47%2C38727%2C1731422599737.1731422601050 so closing down 2024-11-12T14:43:37,608 DEBUG [RS_LOG_REPLAY_OPS-regionserver/3945e1ca6f47:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-12T14:43:37,608 INFO [RS_LOG_REPLAY_OPS-regionserver/3945e1ca6f47:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.OutputSink(145): 3 split writer threads finished 2024-11-12T14:43:37,608 INFO [RS_LOG_REPLAY_OPS-regionserver/3945e1ca6f47:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(425): Processed 6 edits across 0 Regions in 9 ms; skipped=6; WAL=hdfs://localhost:40559/user/jenkins/test-data/41a388f8-1764-825e-e285-25964d37c9f3/WALs/3945e1ca6f47,38727,1731422599737-splitting/3945e1ca6f47%2C38727%2C1731422599737.1731422601050, size=2.2 K, length=2288, corrupted=false, cancelled=false 2024-11-12T14:43:37,608 DEBUG [RS_LOG_REPLAY_OPS-regionserver/3945e1ca6f47:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(428): Completed split of hdfs://localhost:40559/user/jenkins/test-data/41a388f8-1764-825e-e285-25964d37c9f3/WALs/3945e1ca6f47,38727,1731422599737-splitting/3945e1ca6f47%2C38727%2C1731422599737.1731422601050, journal: Splitting hdfs://localhost:40559/user/jenkins/test-data/41a388f8-1764-825e-e285-25964d37c9f3/WALs/3945e1ca6f47,38727,1731422599737-splitting/3945e1ca6f47%2C38727%2C1731422599737.1731422601050, size=2.2 K (2288bytes) at 1731422617558Finishing writing output for hdfs://localhost:40559/user/jenkins/test-data/41a388f8-1764-825e-e285-25964d37c9f3/WALs/3945e1ca6f47,38727,1731422599737-splitting/3945e1ca6f47%2C38727%2C1731422599737.1731422601050 so closing down at 1731422617608 (+50 ms)3 split writer threads finished at 1731422617608Processed 6 edits across 0 Regions in 9 ms; skipped=6; WAL=hdfs://localhost:40559/user/jenkins/test-data/41a388f8-1764-825e-e285-25964d37c9f3/WALs/3945e1ca6f47,38727,1731422599737-splitting/3945e1ca6f47%2C38727%2C1731422599737.1731422601050, size=2.2 K, length=2288, corrupted=false, cancelled=false at 1731422617608 2024-11-12T14:43:37,608 DEBUG [RS_LOG_REPLAY_OPS-regionserver/3945e1ca6f47:0-0 {event_type=RS_LOG_REPLAY, pid=15}] regionserver.SplitLogWorker(218): Done splitting WAL hdfs://localhost:40559/user/jenkins/test-data/41a388f8-1764-825e-e285-25964d37c9f3/WALs/3945e1ca6f47,38727,1731422599737-splitting/3945e1ca6f47%2C38727%2C1731422599737.1731422601050 2024-11-12T14:43:37,610 DEBUG [RS_LOG_REPLAY_OPS-regionserver/3945e1ca6f47:0-0 {event_type=RS_LOG_REPLAY, pid=15}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=15 2024-11-12T14:43:37,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34233 {}] master.HMaster(4169): Remote procedure done, pid=15 2024-11-12T14:43:37,617 INFO [PEWorker-4 {}] wal.WALSplitUtil(143): Moved hdfs://localhost:40559/user/jenkins/test-data/41a388f8-1764-825e-e285-25964d37c9f3/WALs/3945e1ca6f47,38727,1731422599737-splitting/3945e1ca6f47%2C38727%2C1731422599737.1731422601050 to hdfs://localhost:40559/user/jenkins/test-data/41a388f8-1764-825e-e285-25964d37c9f3/oldWALs 2024-11-12T14:43:37,620 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=15, resume processing ppid=14 2024-11-12T14:43:37,621 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, ppid=14, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3945e1ca6f47%2C38727%2C1731422599737.1731422601050, worker=3945e1ca6f47,33387,1731422599699 in 245 msec 2024-11-12T14:43:37,622 DEBUG [PEWorker-5 {}] master.SplitWALManager(172): Release split WAL worker=3945e1ca6f47,33387,1731422599699 2024-11-12T14:43:37,625 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-11-12T14:43:37,625 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; SplitWALProcedure 3945e1ca6f47%2C38727%2C1731422599737.1731422601050, worker=3945e1ca6f47,33387,1731422599699 in 262 msec 2024-11-12T14:43:37,627 INFO [PEWorker-1 {}] master.SplitLogManager(171): hdfs://localhost:40559/user/jenkins/test-data/41a388f8-1764-825e-e285-25964d37c9f3/WALs/3945e1ca6f47,38727,1731422599737-splitting dir is empty, no logs to split. 2024-11-12T14:43:37,627 INFO [PEWorker-1 {}] master.SplitWALManager(105): 3945e1ca6f47,38727,1731422599737 WAL count=0, meta=false 2024-11-12T14:43:37,628 DEBUG [PEWorker-1 {}] procedure.ServerCrashProcedure(329): Check if 3945e1ca6f47,38727,1731422599737 WAL splitting is done? wals=0, meta=false 2024-11-12T14:43:37,630 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(321): Remove WAL directory for 3945e1ca6f47,38727,1731422599737 failed, ignore...File hdfs://localhost:40559/user/jenkins/test-data/41a388f8-1764-825e-e285-25964d37c9f3/WALs/3945e1ca6f47,38727,1731422599737-splitting does not exist. 2024-11-12T14:43:37,633 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=06dfc67af613562882ee0461d2aa7115, ASSIGN}] 2024-11-12T14:43:37,635 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=06dfc67af613562882ee0461d2aa7115, ASSIGN 2024-11-12T14:43:37,636 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=06dfc67af613562882ee0461d2aa7115, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-12T14:43:37,766 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1731422614808.06dfc67af613562882ee0461d2aa7115., hostname=3945e1ca6f47,38727,1731422599737, seqNum=18] 2024-11-12T14:43:37,766 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] ipc.AbstractRpcClient(357): Not trying to connect to 3945e1ca6f47:38727 this server is in the failed servers list 2024-11-12T14:43:37,767 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1731422614808.06dfc67af613562882ee0461d2aa7115., hostname=3945e1ca6f47,38727,1731422599737, seqNum=18 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1731422614808.06dfc67af613562882ee0461d2aa7115., hostname=3945e1ca6f47,38727,1731422599737, seqNum=18, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=3945e1ca6f47:38727 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 3945e1ca6f47:38727 2024-11-12T14:43:37,767 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1731422614808.06dfc67af613562882ee0461d2aa7115., hostname=3945e1ca6f47,38727,1731422599737, seqNum=18 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 3945e1ca6f47:38727 2024-11-12T14:43:37,767 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testReplayEditsAfterRegionMovedWithMultiCF,,1731422614808.06dfc67af613562882ee0461d2aa7115., hostname=3945e1ca6f47,38727,1731422599737, seqNum=18 from cache 2024-11-12T14:43:37,787 DEBUG [3945e1ca6f47:34233 {}] balancer.BalancerClusterState(204): Hosts are {3945e1ca6f47=0} racks are {/default-rack=0} 2024-11-12T14:43:37,787 DEBUG [3945e1ca6f47:34233 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-12T14:43:37,787 DEBUG [3945e1ca6f47:34233 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-12T14:43:37,787 DEBUG [3945e1ca6f47:34233 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-12T14:43:37,787 DEBUG [3945e1ca6f47:34233 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-12T14:43:37,787 INFO [3945e1ca6f47:34233 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-12T14:43:37,787 INFO [3945e1ca6f47:34233 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-12T14:43:37,788 DEBUG [3945e1ca6f47:34233 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-12T14:43:37,788 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=06dfc67af613562882ee0461d2aa7115, regionState=OPENING, regionLocation=3945e1ca6f47,41431,1731422599613 2024-11-12T14:43:37,791 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=06dfc67af613562882ee0461d2aa7115, ASSIGN because future has completed 2024-11-12T14:43:37,791 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=17, ppid=16, state=RUNNABLE, hasLock=false; OpenRegionProcedure 06dfc67af613562882ee0461d2aa7115, server=3945e1ca6f47,41431,1731422599613}] 2024-11-12T14:43:37,950 INFO [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1731422614808.06dfc67af613562882ee0461d2aa7115. 2024-11-12T14:43:37,950 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(7752): Opening region: {ENCODED => 06dfc67af613562882ee0461d2aa7115, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1731422614808.06dfc67af613562882ee0461d2aa7115.', STARTKEY => '', ENDKEY => ''} 2024-11-12T14:43:37,950 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF 06dfc67af613562882ee0461d2aa7115 2024-11-12T14:43:37,950 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1731422614808.06dfc67af613562882ee0461d2aa7115.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T14:43:37,950 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(7794): checking encryption for 06dfc67af613562882ee0461d2aa7115 2024-11-12T14:43:37,950 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(7797): checking classloading for 06dfc67af613562882ee0461d2aa7115 2024-11-12T14:43:37,952 INFO [StoreOpener-06dfc67af613562882ee0461d2aa7115-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 06dfc67af613562882ee0461d2aa7115 2024-11-12T14:43:37,953 INFO [StoreOpener-06dfc67af613562882ee0461d2aa7115-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 06dfc67af613562882ee0461d2aa7115 columnFamilyName cf1 2024-11-12T14:43:37,954 DEBUG [StoreOpener-06dfc67af613562882ee0461d2aa7115-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:43:37,963 DEBUG [StoreOpener-06dfc67af613562882ee0461d2aa7115-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:40559/user/jenkins/test-data/41a388f8-1764-825e-e285-25964d37c9f3/data/default/testReplayEditsAfterRegionMovedWithMultiCF/06dfc67af613562882ee0461d2aa7115/cf1/44a277ae66ab433ba4efd1edc131dfb4 2024-11-12T14:43:37,963 INFO [StoreOpener-06dfc67af613562882ee0461d2aa7115-1 {}] regionserver.HStore(327): Store=06dfc67af613562882ee0461d2aa7115/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T14:43:37,963 INFO [StoreOpener-06dfc67af613562882ee0461d2aa7115-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 06dfc67af613562882ee0461d2aa7115 2024-11-12T14:43:37,965 INFO [StoreOpener-06dfc67af613562882ee0461d2aa7115-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 06dfc67af613562882ee0461d2aa7115 columnFamilyName cf2 2024-11-12T14:43:37,965 DEBUG [StoreOpener-06dfc67af613562882ee0461d2aa7115-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:43:37,976 DEBUG [StoreOpener-06dfc67af613562882ee0461d2aa7115-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:40559/user/jenkins/test-data/41a388f8-1764-825e-e285-25964d37c9f3/data/default/testReplayEditsAfterRegionMovedWithMultiCF/06dfc67af613562882ee0461d2aa7115/cf2/49854fb177e64856a73947a38e838905 2024-11-12T14:43:37,976 INFO [StoreOpener-06dfc67af613562882ee0461d2aa7115-1 {}] regionserver.HStore(327): Store=06dfc67af613562882ee0461d2aa7115/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T14:43:37,976 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1038): replaying wal for 06dfc67af613562882ee0461d2aa7115 2024-11-12T14:43:37,977 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40559/user/jenkins/test-data/41a388f8-1764-825e-e285-25964d37c9f3/data/default/testReplayEditsAfterRegionMovedWithMultiCF/06dfc67af613562882ee0461d2aa7115 2024-11-12T14:43:37,979 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40559/user/jenkins/test-data/41a388f8-1764-825e-e285-25964d37c9f3/data/default/testReplayEditsAfterRegionMovedWithMultiCF/06dfc67af613562882ee0461d2aa7115 2024-11-12T14:43:37,979 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1048): stopping wal replay for 06dfc67af613562882ee0461d2aa7115 2024-11-12T14:43:37,980 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1060): Cleaning up temporary data for 06dfc67af613562882ee0461d2aa7115 2024-11-12T14:43:37,980 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-11-12T14:43:37,982 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1093): writing seq id for 06dfc67af613562882ee0461d2aa7115 2024-11-12T14:43:37,983 INFO [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1114): Opened 06dfc67af613562882ee0461d2aa7115; next sequenceid=18; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61295402, jitterRate=-0.08662733435630798}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-11-12T14:43:37,983 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 06dfc67af613562882ee0461d2aa7115 2024-11-12T14:43:37,984 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1006): Region open journal for 06dfc67af613562882ee0461d2aa7115: Running coprocessor pre-open hook at 1731422617950Writing region info on filesystem at 1731422617950Initializing all the Stores at 1731422617951 (+1 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731422617952 (+1 ms)Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731422617952Cleaning up temporary data from old regions at 1731422617980 (+28 ms)Running coprocessor post-open hooks at 1731422617983 (+3 ms)Region opened successfully at 1731422617984 (+1 ms) 2024-11-12T14:43:37,985 INFO [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1731422614808.06dfc67af613562882ee0461d2aa7115., pid=17, masterSystemTime=1731422617944 2024-11-12T14:43:37,988 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1731422614808.06dfc67af613562882ee0461d2aa7115. 2024-11-12T14:43:37,988 INFO [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1731422614808.06dfc67af613562882ee0461d2aa7115. 2024-11-12T14:43:37,990 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=06dfc67af613562882ee0461d2aa7115, regionState=OPEN, openSeqNum=18, regionLocation=3945e1ca6f47,41431,1731422599613 2024-11-12T14:43:37,993 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=17, ppid=16, state=RUNNABLE, hasLock=false; OpenRegionProcedure 06dfc67af613562882ee0461d2aa7115, server=3945e1ca6f47,41431,1731422599613 because future has completed 2024-11-12T14:43:37,998 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=17, resume processing ppid=16 2024-11-12T14:43:37,998 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, ppid=16, state=SUCCESS, hasLock=false; OpenRegionProcedure 06dfc67af613562882ee0461d2aa7115, server=3945e1ca6f47,41431,1731422599613 in 204 msec 2024-11-12T14:43:38,000 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=13 2024-11-12T14:43:38,000 INFO [PEWorker-4 {}] procedure.ServerCrashProcedure(291): removed crashed server 3945e1ca6f47,38727,1731422599737 after splitting done 2024-11-12T14:43:38,001 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=13, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=06dfc67af613562882ee0461d2aa7115, ASSIGN in 365 msec 2024-11-12T14:43:38,004 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; ServerCrashProcedure 3945e1ca6f47,38727,1731422599737, splitWal=true, meta=false in 654 msec 2024-11-12T14:43:38,276 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1731422614808.06dfc67af613562882ee0461d2aa7115., hostname=3945e1ca6f47,41431,1731422599613, seqNum=18] 2024-11-12T14:43:38,297 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsAfterRegionMovedWithMultiCF Thread=413 (was 417), OpenFileDescriptor=1051 (was 1023) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=260 (was 283), ProcessCount=11 (was 11), AvailableMemoryMB=8419 (was 7422) - AvailableMemoryMB LEAK? - 2024-11-12T14:43:38,299 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1051 is superior to 1024 2024-11-12T14:43:38,318 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsAfterPartialFlush Thread=413, OpenFileDescriptor=1051, MaxFileDescriptor=1048576, SystemLoadAverage=260, ProcessCount=11, AvailableMemoryMB=8418 2024-11-12T14:43:38,318 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1051 is superior to 1024 2024-11-12T14:43:38,342 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-12T14:43:38,345 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-12T14:43:38,346 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-12T14:43:38,350 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-75023884, suffix=, logDir=hdfs://localhost:40559/hbase/WALs/hregion-75023884, archiveDir=hdfs://localhost:40559/hbase/oldWALs, maxLogs=32 2024-11-12T14:43:38,369 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-75023884/hregion-75023884.1731422618350, exclude list is [], retry=0 2024-11-12T14:43:38,374 DEBUG [AsyncFSWAL-20-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37395,DS-7317e4d9-cef6-4b85-a947-106c3c917da9,DISK] 2024-11-12T14:43:38,374 DEBUG [AsyncFSWAL-20-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36175,DS-31e7ba0a-e37c-4fe6-8d2a-a4846cefa408,DISK] 2024-11-12T14:43:38,375 DEBUG [AsyncFSWAL-20-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45975,DS-80b047b6-34d2-41c0-99c3-5cd61ca40075,DISK] 2024-11-12T14:43:38,422 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-75023884/hregion-75023884.1731422618350 2024-11-12T14:43:38,423 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36027:36027),(127.0.0.1/127.0.0.1:40227:40227),(127.0.0.1/127.0.0.1:43643:43643)] 2024-11-12T14:43:38,423 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 12438d1e8547c512e84ac40b3c7da275, NAME => 'testReplayEditsWrittenViaHRegion,,1731422618343.12438d1e8547c512e84ac40b3c7da275.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenViaHRegion', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40559/hbase 2024-11-12T14:43:38,451 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36175 is added to blk_1073741893_1071 (size=67) 2024-11-12T14:43:38,452 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37395 is added to blk_1073741893_1071 (size=67) 2024-11-12T14:43:38,452 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45975 is added to blk_1073741893_1071 (size=67) 2024-11-12T14:43:38,453 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1731422618343.12438d1e8547c512e84ac40b3c7da275.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T14:43:38,454 INFO [StoreOpener-12438d1e8547c512e84ac40b3c7da275-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 12438d1e8547c512e84ac40b3c7da275 2024-11-12T14:43:38,456 INFO [StoreOpener-12438d1e8547c512e84ac40b3c7da275-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 12438d1e8547c512e84ac40b3c7da275 columnFamilyName a 2024-11-12T14:43:38,456 DEBUG [StoreOpener-12438d1e8547c512e84ac40b3c7da275-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:43:38,457 INFO [StoreOpener-12438d1e8547c512e84ac40b3c7da275-1 {}] regionserver.HStore(327): Store=12438d1e8547c512e84ac40b3c7da275/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T14:43:38,457 INFO [StoreOpener-12438d1e8547c512e84ac40b3c7da275-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 12438d1e8547c512e84ac40b3c7da275 2024-11-12T14:43:38,459 INFO [StoreOpener-12438d1e8547c512e84ac40b3c7da275-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 12438d1e8547c512e84ac40b3c7da275 columnFamilyName b 2024-11-12T14:43:38,459 DEBUG [StoreOpener-12438d1e8547c512e84ac40b3c7da275-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:43:38,463 INFO [StoreOpener-12438d1e8547c512e84ac40b3c7da275-1 {}] regionserver.HStore(327): Store=12438d1e8547c512e84ac40b3c7da275/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T14:43:38,463 INFO [StoreOpener-12438d1e8547c512e84ac40b3c7da275-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 12438d1e8547c512e84ac40b3c7da275 2024-11-12T14:43:38,465 INFO [StoreOpener-12438d1e8547c512e84ac40b3c7da275-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 12438d1e8547c512e84ac40b3c7da275 columnFamilyName c 2024-11-12T14:43:38,465 DEBUG [StoreOpener-12438d1e8547c512e84ac40b3c7da275-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:43:38,466 INFO [StoreOpener-12438d1e8547c512e84ac40b3c7da275-1 {}] regionserver.HStore(327): Store=12438d1e8547c512e84ac40b3c7da275/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T14:43:38,466 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 12438d1e8547c512e84ac40b3c7da275 2024-11-12T14:43:38,467 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40559/hbase/data/default/testReplayEditsWrittenViaHRegion/12438d1e8547c512e84ac40b3c7da275 2024-11-12T14:43:38,468 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40559/hbase/data/default/testReplayEditsWrittenViaHRegion/12438d1e8547c512e84ac40b3c7da275 2024-11-12T14:43:38,469 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 12438d1e8547c512e84ac40b3c7da275 2024-11-12T14:43:38,469 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 12438d1e8547c512e84ac40b3c7da275 2024-11-12T14:43:38,470 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-12T14:43:38,472 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 12438d1e8547c512e84ac40b3c7da275 2024-11-12T14:43:38,475 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40559/hbase/data/default/testReplayEditsWrittenViaHRegion/12438d1e8547c512e84ac40b3c7da275/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-12T14:43:38,477 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 12438d1e8547c512e84ac40b3c7da275; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62534930, jitterRate=-0.06815692782402039}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-12T14:43:38,478 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 12438d1e8547c512e84ac40b3c7da275: Writing region info on filesystem at 1731422618453Initializing all the Stores at 1731422618454 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731422618454Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731422618454Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731422618454Cleaning up temporary data from old regions at 1731422618470 (+16 ms)Region opened successfully at 1731422618478 (+8 ms) 2024-11-12T14:43:38,478 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 12438d1e8547c512e84ac40b3c7da275, disabling compactions & flushes 2024-11-12T14:43:38,478 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1731422618343.12438d1e8547c512e84ac40b3c7da275. 2024-11-12T14:43:38,478 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1731422618343.12438d1e8547c512e84ac40b3c7da275. 2024-11-12T14:43:38,478 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1731422618343.12438d1e8547c512e84ac40b3c7da275. after waiting 0 ms 2024-11-12T14:43:38,478 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1731422618343.12438d1e8547c512e84ac40b3c7da275. 2024-11-12T14:43:38,479 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1731422618343.12438d1e8547c512e84ac40b3c7da275. 2024-11-12T14:43:38,479 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 12438d1e8547c512e84ac40b3c7da275: Waiting for close lock at 1731422618478Disabling compacts and flushes for region at 1731422618478Disabling writes for close at 1731422618478Writing region close event to WAL at 1731422618479 (+1 ms)Closed at 1731422618479 2024-11-12T14:43:38,482 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37395 is added to blk_1073741892_1070 (size=175) 2024-11-12T14:43:38,482 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36175 is added to blk_1073741892_1070 (size=175) 2024-11-12T14:43:38,482 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45975 is added to blk_1073741892_1070 (size=175) 2024-11-12T14:43:38,486 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-12T14:43:38,486 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-75023884:(num 1731422618350) 2024-11-12T14:43:38,486 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-12T14:43:38,489 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:40559/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731422618341, archiveDir=hdfs://localhost:40559/hbase/oldWALs, maxLogs=32 2024-11-12T14:43:38,508 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731422618341/wal.1731422618489, exclude list is [], retry=0 2024-11-12T14:43:38,511 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37395,DS-7317e4d9-cef6-4b85-a947-106c3c917da9,DISK] 2024-11-12T14:43:38,511 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45975,DS-80b047b6-34d2-41c0-99c3-5cd61ca40075,DISK] 2024-11-12T14:43:38,512 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36175,DS-31e7ba0a-e37c-4fe6-8d2a-a4846cefa408,DISK] 2024-11-12T14:43:38,540 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731422618341/wal.1731422618489 2024-11-12T14:43:38,540 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40227:40227),(127.0.0.1/127.0.0.1:43643:43643),(127.0.0.1/127.0.0.1:36027:36027)] 2024-11-12T14:43:38,540 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 12438d1e8547c512e84ac40b3c7da275, NAME => 'testReplayEditsWrittenViaHRegion,,1731422618343.12438d1e8547c512e84ac40b3c7da275.', STARTKEY => '', ENDKEY => ''} 2024-11-12T14:43:38,541 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1731422618343.12438d1e8547c512e84ac40b3c7da275.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T14:43:38,541 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 12438d1e8547c512e84ac40b3c7da275 2024-11-12T14:43:38,541 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 12438d1e8547c512e84ac40b3c7da275 2024-11-12T14:43:38,545 INFO [StoreOpener-12438d1e8547c512e84ac40b3c7da275-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 12438d1e8547c512e84ac40b3c7da275 2024-11-12T14:43:38,546 INFO [StoreOpener-12438d1e8547c512e84ac40b3c7da275-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 12438d1e8547c512e84ac40b3c7da275 columnFamilyName a 2024-11-12T14:43:38,547 DEBUG [StoreOpener-12438d1e8547c512e84ac40b3c7da275-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:43:38,547 INFO [StoreOpener-12438d1e8547c512e84ac40b3c7da275-1 {}] regionserver.HStore(327): Store=12438d1e8547c512e84ac40b3c7da275/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T14:43:38,548 INFO [StoreOpener-12438d1e8547c512e84ac40b3c7da275-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 12438d1e8547c512e84ac40b3c7da275 2024-11-12T14:43:38,549 INFO [StoreOpener-12438d1e8547c512e84ac40b3c7da275-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 12438d1e8547c512e84ac40b3c7da275 columnFamilyName b 2024-11-12T14:43:38,549 DEBUG [StoreOpener-12438d1e8547c512e84ac40b3c7da275-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:43:38,550 INFO [StoreOpener-12438d1e8547c512e84ac40b3c7da275-1 {}] regionserver.HStore(327): Store=12438d1e8547c512e84ac40b3c7da275/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T14:43:38,550 INFO [StoreOpener-12438d1e8547c512e84ac40b3c7da275-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 12438d1e8547c512e84ac40b3c7da275 2024-11-12T14:43:38,551 INFO [StoreOpener-12438d1e8547c512e84ac40b3c7da275-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 12438d1e8547c512e84ac40b3c7da275 columnFamilyName c 2024-11-12T14:43:38,551 DEBUG [StoreOpener-12438d1e8547c512e84ac40b3c7da275-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:43:38,552 INFO [StoreOpener-12438d1e8547c512e84ac40b3c7da275-1 {}] regionserver.HStore(327): Store=12438d1e8547c512e84ac40b3c7da275/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T14:43:38,552 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 12438d1e8547c512e84ac40b3c7da275 2024-11-12T14:43:38,553 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40559/hbase/data/default/testReplayEditsWrittenViaHRegion/12438d1e8547c512e84ac40b3c7da275 2024-11-12T14:43:38,555 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40559/hbase/data/default/testReplayEditsWrittenViaHRegion/12438d1e8547c512e84ac40b3c7da275 2024-11-12T14:43:38,561 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 12438d1e8547c512e84ac40b3c7da275 2024-11-12T14:43:38,561 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 12438d1e8547c512e84ac40b3c7da275 2024-11-12T14:43:38,562 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-12T14:43:38,564 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 12438d1e8547c512e84ac40b3c7da275 2024-11-12T14:43:38,567 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 12438d1e8547c512e84ac40b3c7da275; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66654694, jitterRate=-0.00676766037940979}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-12T14:43:38,568 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 12438d1e8547c512e84ac40b3c7da275: Writing region info on filesystem at 1731422618541Initializing all the Stores at 1731422618542 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731422618542Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731422618545 (+3 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731422618545Cleaning up temporary data from old regions at 1731422618561 (+16 ms)Region opened successfully at 1731422618568 (+7 ms) 2024-11-12T14:43:38,627 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 12438d1e8547c512e84ac40b3c7da275 3/3 column families, dataSize=2.55 KB heapSize=5.44 KB 2024-11-12T14:43:38,653 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40559/hbase/data/default/testReplayEditsWrittenViaHRegion/12438d1e8547c512e84ac40b3c7da275/.tmp/a/81c2fde7d4f14b4d9fd1ff52a1089bff is 91, key is testReplayEditsWrittenViaHRegion/a:x0/1731422618568/Put/seqid=0 2024-11-12T14:43:38,670 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45975 is added to blk_1073741895_1073 (size=5958) 2024-11-12T14:43:38,670 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37395 is added to blk_1073741895_1073 (size=5958) 2024-11-12T14:43:38,671 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36175 is added to blk_1073741895_1073 (size=5958) 2024-11-12T14:43:38,674 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:40559/hbase/data/default/testReplayEditsWrittenViaHRegion/12438d1e8547c512e84ac40b3c7da275/.tmp/a/81c2fde7d4f14b4d9fd1ff52a1089bff 2024-11-12T14:43:38,704 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40559/hbase/data/default/testReplayEditsWrittenViaHRegion/12438d1e8547c512e84ac40b3c7da275/.tmp/b/7847c233e2fb4f5899b204957e985e66 is 91, key is testReplayEditsWrittenViaHRegion/b:x0/1731422618585/Put/seqid=0 2024-11-12T14:43:38,725 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37395 is added to blk_1073741896_1074 (size=5958) 2024-11-12T14:43:38,729 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45975 is added to blk_1073741896_1074 (size=5958) 2024-11-12T14:43:38,730 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36175 is added to blk_1073741896_1074 (size=5958) 2024-11-12T14:43:38,731 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:40559/hbase/data/default/testReplayEditsWrittenViaHRegion/12438d1e8547c512e84ac40b3c7da275/.tmp/b/7847c233e2fb4f5899b204957e985e66 2024-11-12T14:43:38,762 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40559/hbase/data/default/testReplayEditsWrittenViaHRegion/12438d1e8547c512e84ac40b3c7da275/.tmp/c/1e21f47a2bb84a5897a5864ca5fb4e44 is 91, key is testReplayEditsWrittenViaHRegion/c:x0/1731422618603/Put/seqid=0 2024-11-12T14:43:38,777 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45975 is added to blk_1073741897_1075 (size=5958) 2024-11-12T14:43:38,777 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36175 is added to blk_1073741897_1075 (size=5958) 2024-11-12T14:43:38,779 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37395 is added to blk_1073741897_1075 (size=5958) 2024-11-12T14:43:38,780 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:40559/hbase/data/default/testReplayEditsWrittenViaHRegion/12438d1e8547c512e84ac40b3c7da275/.tmp/c/1e21f47a2bb84a5897a5864ca5fb4e44 2024-11-12T14:43:38,795 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40559/hbase/data/default/testReplayEditsWrittenViaHRegion/12438d1e8547c512e84ac40b3c7da275/.tmp/a/81c2fde7d4f14b4d9fd1ff52a1089bff as hdfs://localhost:40559/hbase/data/default/testReplayEditsWrittenViaHRegion/12438d1e8547c512e84ac40b3c7da275/a/81c2fde7d4f14b4d9fd1ff52a1089bff 2024-11-12T14:43:38,802 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40559/hbase/data/default/testReplayEditsWrittenViaHRegion/12438d1e8547c512e84ac40b3c7da275/a/81c2fde7d4f14b4d9fd1ff52a1089bff, entries=10, sequenceid=33, filesize=5.8 K 2024-11-12T14:43:38,803 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40559/hbase/data/default/testReplayEditsWrittenViaHRegion/12438d1e8547c512e84ac40b3c7da275/.tmp/b/7847c233e2fb4f5899b204957e985e66 as hdfs://localhost:40559/hbase/data/default/testReplayEditsWrittenViaHRegion/12438d1e8547c512e84ac40b3c7da275/b/7847c233e2fb4f5899b204957e985e66 2024-11-12T14:43:38,811 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40559/hbase/data/default/testReplayEditsWrittenViaHRegion/12438d1e8547c512e84ac40b3c7da275/b/7847c233e2fb4f5899b204957e985e66, entries=10, sequenceid=33, filesize=5.8 K 2024-11-12T14:43:38,812 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40559/hbase/data/default/testReplayEditsWrittenViaHRegion/12438d1e8547c512e84ac40b3c7da275/.tmp/c/1e21f47a2bb84a5897a5864ca5fb4e44 as hdfs://localhost:40559/hbase/data/default/testReplayEditsWrittenViaHRegion/12438d1e8547c512e84ac40b3c7da275/c/1e21f47a2bb84a5897a5864ca5fb4e44 2024-11-12T14:43:38,821 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40559/hbase/data/default/testReplayEditsWrittenViaHRegion/12438d1e8547c512e84ac40b3c7da275/c/1e21f47a2bb84a5897a5864ca5fb4e44, entries=10, sequenceid=33, filesize=5.8 K 2024-11-12T14:43:38,823 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~2.55 KB/2610, heapSize ~5.39 KB/5520, currentSize=0 B/0 for 12438d1e8547c512e84ac40b3c7da275 in 196ms, sequenceid=33, compaction requested=false 2024-11-12T14:43:38,823 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 12438d1e8547c512e84ac40b3c7da275: 2024-11-12T14:43:38,823 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 12438d1e8547c512e84ac40b3c7da275, disabling compactions & flushes 2024-11-12T14:43:38,823 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1731422618343.12438d1e8547c512e84ac40b3c7da275. 2024-11-12T14:43:38,823 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1731422618343.12438d1e8547c512e84ac40b3c7da275. 2024-11-12T14:43:38,824 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1731422618343.12438d1e8547c512e84ac40b3c7da275. after waiting 0 ms 2024-11-12T14:43:38,824 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1731422618343.12438d1e8547c512e84ac40b3c7da275. 2024-11-12T14:43:38,837 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1731422618343.12438d1e8547c512e84ac40b3c7da275. 2024-11-12T14:43:38,838 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 12438d1e8547c512e84ac40b3c7da275: Waiting for close lock at 1731422618823Disabling compacts and flushes for region at 1731422618823Disabling writes for close at 1731422618824 (+1 ms)Writing region close event to WAL at 1731422618837 (+13 ms)Closed at 1731422618837 2024-11-12T14:43:38,853 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45975 is added to blk_1073741894_1072 (size=6311) 2024-11-12T14:43:38,853 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36175 is added to blk_1073741894_1072 (size=6311) 2024-11-12T14:43:38,853 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37395 is added to blk_1073741894_1072 (size=6311) 2024-11-12T14:43:38,862 DEBUG [Time-limited test {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40559/hbase/data/default/testReplayEditsWrittenViaHRegion/12438d1e8547c512e84ac40b3c7da275/b/7847c233e2fb4f5899b204957e985e66 to hdfs://localhost:40559/hbase/archive/data/default/testReplayEditsWrittenViaHRegion/12438d1e8547c512e84ac40b3c7da275/b/7847c233e2fb4f5899b204957e985e66 2024-11-12T14:43:38,893 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:40559/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731422618341/wal.1731422618489, size=6.2 K (6311bytes) 2024-11-12T14:43:38,893 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:40559/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731422618341/wal.1731422618489 2024-11-12T14:43:38,894 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:40559/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731422618341/wal.1731422618489 after 1ms 2024-11-12T14:43:38,945 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-12T14:43:38,945 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:40559/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731422618341/wal.1731422618489 took 53ms 2024-11-12T14:43:38,951 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:40559/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731422618341/wal.1731422618489 so closing down 2024-11-12T14:43:38,951 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-12T14:43:38,955 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1731422618489.temp 2024-11-12T14:43:38,989 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:40559/hbase/data/default/testReplayEditsWrittenViaHRegion/12438d1e8547c512e84ac40b3c7da275/recovered.edits/0000000000000000003-wal.1731422618489.temp 2024-11-12T14:43:38,989 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-12T14:43:39,010 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37395 is added to blk_1073741898_1076 (size=5575) 2024-11-12T14:43:39,011 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45975 is added to blk_1073741898_1076 (size=5575) 2024-11-12T14:43:39,011 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36175 is added to blk_1073741898_1076 (size=5575) 2024-11-12T14:43:39,012 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:40559/hbase/data/default/testReplayEditsWrittenViaHRegion/12438d1e8547c512e84ac40b3c7da275/recovered.edits/0000000000000000003-wal.1731422618489.temp (wrote 30 edits, skipped 0 edits in 0 ms) 2024-11-12T14:43:39,013 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:40559/hbase/data/default/testReplayEditsWrittenViaHRegion/12438d1e8547c512e84ac40b3c7da275/recovered.edits/0000000000000000003-wal.1731422618489.temp to hdfs://localhost:40559/hbase/data/default/testReplayEditsWrittenViaHRegion/12438d1e8547c512e84ac40b3c7da275/recovered.edits/0000000000000000032 2024-11-12T14:43:39,014 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 32 edits across 1 Regions in 66 ms; skipped=2; WAL=hdfs://localhost:40559/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731422618341/wal.1731422618489, size=6.2 K, length=6311, corrupted=false, cancelled=false 2024-11-12T14:43:39,014 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:40559/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731422618341/wal.1731422618489, journal: Splitting hdfs://localhost:40559/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731422618341/wal.1731422618489, size=6.2 K (6311bytes) at 1731422618893Finishing writing output for hdfs://localhost:40559/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731422618341/wal.1731422618489 so closing down at 1731422618951 (+58 ms)Creating recovered edits writer path=hdfs://localhost:40559/hbase/data/default/testReplayEditsWrittenViaHRegion/12438d1e8547c512e84ac40b3c7da275/recovered.edits/0000000000000000003-wal.1731422618489.temp at 1731422618989 (+38 ms)3 split writer threads finished at 1731422618989Closed recovered edits writer path=hdfs://localhost:40559/hbase/data/default/testReplayEditsWrittenViaHRegion/12438d1e8547c512e84ac40b3c7da275/recovered.edits/0000000000000000003-wal.1731422618489.temp (wrote 30 edits, skipped 0 edits in 0 ms) at 1731422619012 (+23 ms)Rename recovered edits hdfs://localhost:40559/hbase/data/default/testReplayEditsWrittenViaHRegion/12438d1e8547c512e84ac40b3c7da275/recovered.edits/0000000000000000003-wal.1731422618489.temp to hdfs://localhost:40559/hbase/data/default/testReplayEditsWrittenViaHRegion/12438d1e8547c512e84ac40b3c7da275/recovered.edits/0000000000000000032 at 1731422619013 (+1 ms)Processed 32 edits across 1 Regions in 66 ms; skipped=2; WAL=hdfs://localhost:40559/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731422618341/wal.1731422618489, size=6.2 K, length=6311, corrupted=false, cancelled=false at 1731422619014 (+1 ms) 2024-11-12T14:43:39,016 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:40559/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731422618341/wal.1731422618489 to hdfs://localhost:40559/hbase/oldWALs/wal.1731422618489 2024-11-12T14:43:39,016 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:40559/hbase/data/default/testReplayEditsWrittenViaHRegion/12438d1e8547c512e84ac40b3c7da275/recovered.edits/0000000000000000032 2024-11-12T14:43:39,017 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-12T14:43:39,019 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:40559/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731422618341, archiveDir=hdfs://localhost:40559/hbase/oldWALs, maxLogs=32 2024-11-12T14:43:39,039 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731422618341/wal.1731422619019, exclude list is [], retry=0 2024-11-12T14:43:39,042 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36175,DS-31e7ba0a-e37c-4fe6-8d2a-a4846cefa408,DISK] 2024-11-12T14:43:39,043 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45975,DS-80b047b6-34d2-41c0-99c3-5cd61ca40075,DISK] 2024-11-12T14:43:39,043 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37395,DS-7317e4d9-cef6-4b85-a947-106c3c917da9,DISK] 2024-11-12T14:43:39,094 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731422618341/wal.1731422619019 2024-11-12T14:43:39,095 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36027:36027),(127.0.0.1/127.0.0.1:43643:43643),(127.0.0.1/127.0.0.1:40227:40227)] 2024-11-12T14:43:39,095 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 12438d1e8547c512e84ac40b3c7da275, NAME => 'testReplayEditsWrittenViaHRegion,,1731422618343.12438d1e8547c512e84ac40b3c7da275.', STARTKEY => '', ENDKEY => ''} 2024-11-12T14:43:39,095 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1731422618343.12438d1e8547c512e84ac40b3c7da275.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T14:43:39,095 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 12438d1e8547c512e84ac40b3c7da275 2024-11-12T14:43:39,095 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 12438d1e8547c512e84ac40b3c7da275 2024-11-12T14:43:39,097 INFO [StoreOpener-12438d1e8547c512e84ac40b3c7da275-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 12438d1e8547c512e84ac40b3c7da275 2024-11-12T14:43:39,099 INFO [StoreOpener-12438d1e8547c512e84ac40b3c7da275-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 12438d1e8547c512e84ac40b3c7da275 columnFamilyName a 2024-11-12T14:43:39,099 DEBUG [StoreOpener-12438d1e8547c512e84ac40b3c7da275-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:43:39,106 DEBUG [StoreOpener-12438d1e8547c512e84ac40b3c7da275-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:40559/hbase/data/default/testReplayEditsWrittenViaHRegion/12438d1e8547c512e84ac40b3c7da275/a/81c2fde7d4f14b4d9fd1ff52a1089bff 2024-11-12T14:43:39,107 INFO [StoreOpener-12438d1e8547c512e84ac40b3c7da275-1 {}] regionserver.HStore(327): Store=12438d1e8547c512e84ac40b3c7da275/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T14:43:39,107 INFO [StoreOpener-12438d1e8547c512e84ac40b3c7da275-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 12438d1e8547c512e84ac40b3c7da275 2024-11-12T14:43:39,109 INFO [StoreOpener-12438d1e8547c512e84ac40b3c7da275-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 12438d1e8547c512e84ac40b3c7da275 columnFamilyName b 2024-11-12T14:43:39,109 DEBUG [StoreOpener-12438d1e8547c512e84ac40b3c7da275-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:43:39,110 INFO [StoreOpener-12438d1e8547c512e84ac40b3c7da275-1 {}] regionserver.HStore(327): Store=12438d1e8547c512e84ac40b3c7da275/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T14:43:39,110 INFO [StoreOpener-12438d1e8547c512e84ac40b3c7da275-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 12438d1e8547c512e84ac40b3c7da275 2024-11-12T14:43:39,111 INFO [StoreOpener-12438d1e8547c512e84ac40b3c7da275-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 12438d1e8547c512e84ac40b3c7da275 columnFamilyName c 2024-11-12T14:43:39,111 DEBUG [StoreOpener-12438d1e8547c512e84ac40b3c7da275-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:43:39,119 DEBUG [StoreOpener-12438d1e8547c512e84ac40b3c7da275-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:40559/hbase/data/default/testReplayEditsWrittenViaHRegion/12438d1e8547c512e84ac40b3c7da275/c/1e21f47a2bb84a5897a5864ca5fb4e44 2024-11-12T14:43:39,119 INFO [StoreOpener-12438d1e8547c512e84ac40b3c7da275-1 {}] regionserver.HStore(327): Store=12438d1e8547c512e84ac40b3c7da275/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T14:43:39,120 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 12438d1e8547c512e84ac40b3c7da275 2024-11-12T14:43:39,121 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40559/hbase/data/default/testReplayEditsWrittenViaHRegion/12438d1e8547c512e84ac40b3c7da275 2024-11-12T14:43:39,123 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:40559/hbase/data/default/testReplayEditsWrittenViaHRegion/12438d1e8547c512e84ac40b3c7da275 2024-11-12T14:43:39,124 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:40559/hbase/data/default/testReplayEditsWrittenViaHRegion/12438d1e8547c512e84ac40b3c7da275/recovered.edits/0000000000000000032 2024-11-12T14:43:39,165 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-12T14:43:39,167 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 10, skipped 20, firstSequenceIdInLog=3, maxSequenceIdInLog=32, path=hdfs://localhost:40559/hbase/data/default/testReplayEditsWrittenViaHRegion/12438d1e8547c512e84ac40b3c7da275/recovered.edits/0000000000000000032 2024-11-12T14:43:39,167 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 12438d1e8547c512e84ac40b3c7da275 3/3 column families, dataSize=870 B heapSize=2.31 KB 2024-11-12T14:43:39,188 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40559/hbase/data/default/testReplayEditsWrittenViaHRegion/12438d1e8547c512e84ac40b3c7da275/.tmp/b/757985ce2d634e9b918008fd4e836862 is 91, key is testReplayEditsWrittenViaHRegion/b:x0/1731422618585/Put/seqid=0 2024-11-12T14:43:39,194 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37395 is added to blk_1073741900_1078 (size=5958) 2024-11-12T14:43:39,195 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45975 is added to blk_1073741900_1078 (size=5958) 2024-11-12T14:43:39,195 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36175 is added to blk_1073741900_1078 (size=5958) 2024-11-12T14:43:39,195 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=32 (bloomFilter=true), to=hdfs://localhost:40559/hbase/data/default/testReplayEditsWrittenViaHRegion/12438d1e8547c512e84ac40b3c7da275/.tmp/b/757985ce2d634e9b918008fd4e836862 2024-11-12T14:43:39,202 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40559/hbase/data/default/testReplayEditsWrittenViaHRegion/12438d1e8547c512e84ac40b3c7da275/.tmp/b/757985ce2d634e9b918008fd4e836862 as hdfs://localhost:40559/hbase/data/default/testReplayEditsWrittenViaHRegion/12438d1e8547c512e84ac40b3c7da275/b/757985ce2d634e9b918008fd4e836862 2024-11-12T14:43:39,208 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40559/hbase/data/default/testReplayEditsWrittenViaHRegion/12438d1e8547c512e84ac40b3c7da275/b/757985ce2d634e9b918008fd4e836862, entries=10, sequenceid=32, filesize=5.8 K 2024-11-12T14:43:39,209 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~870 B/870, heapSize ~1.80 KB/1840, currentSize=0 B/0 for 12438d1e8547c512e84ac40b3c7da275 in 42ms, sequenceid=32, compaction requested=false; wal=null 2024-11-12T14:43:39,210 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:40559/hbase/data/default/testReplayEditsWrittenViaHRegion/12438d1e8547c512e84ac40b3c7da275/recovered.edits/0000000000000000032 2024-11-12T14:43:39,211 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 12438d1e8547c512e84ac40b3c7da275 2024-11-12T14:43:39,211 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 12438d1e8547c512e84ac40b3c7da275 2024-11-12T14:43:39,212 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-12T14:43:39,213 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 12438d1e8547c512e84ac40b3c7da275 2024-11-12T14:43:39,216 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40559/hbase/data/default/testReplayEditsWrittenViaHRegion/12438d1e8547c512e84ac40b3c7da275/recovered.edits/33.seqid, newMaxSeqId=33, maxSeqId=1 2024-11-12T14:43:39,217 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 12438d1e8547c512e84ac40b3c7da275; next sequenceid=34; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63141352, jitterRate=-0.0591205358505249}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-12T14:43:39,217 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 12438d1e8547c512e84ac40b3c7da275: Writing region info on filesystem at 1731422619095Initializing all the Stores at 1731422619096 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731422619096Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731422619097 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731422619097Obtaining lock to block concurrent updates at 1731422619167 (+70 ms)Preparing flush snapshotting stores in 12438d1e8547c512e84ac40b3c7da275 at 1731422619167Finished memstore snapshotting testReplayEditsWrittenViaHRegion,,1731422618343.12438d1e8547c512e84ac40b3c7da275., syncing WAL and waiting on mvcc, flushsize=dataSize=870, getHeapSize=2320, getOffHeapSize=0, getCellsCount=10 at 1731422619168 (+1 ms)Flushing stores of testReplayEditsWrittenViaHRegion,,1731422618343.12438d1e8547c512e84ac40b3c7da275. at 1731422619168Flushing 12438d1e8547c512e84ac40b3c7da275/b: creating writer at 1731422619168Flushing 12438d1e8547c512e84ac40b3c7da275/b: appending metadata at 1731422619187 (+19 ms)Flushing 12438d1e8547c512e84ac40b3c7da275/b: closing flushed file at 1731422619187Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@372c9df8: reopening flushed file at 1731422619201 (+14 ms)Finished flush of dataSize ~870 B/870, heapSize ~1.80 KB/1840, currentSize=0 B/0 for 12438d1e8547c512e84ac40b3c7da275 in 42ms, sequenceid=32, compaction requested=false; wal=null at 1731422619209 (+8 ms)Cleaning up temporary data from old regions at 1731422619211 (+2 ms)Region opened successfully at 1731422619217 (+6 ms) 2024-11-12T14:43:39,247 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsAfterPartialFlush Thread=420 (was 413) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-425049348_22 at /127.0.0.1:38682 [Waiting for operation #36] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-20-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1980882744-172.17.0.2-1731422596004:blk_1073741899_1077, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-425049348_22 at /127.0.0.1:36170 [Waiting for operation #29] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1980882744-172.17.0.2-1731422596004:blk_1073741899_1077, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1980882744-172.17.0.2-1731422596004:blk_1073741899_1077, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-20-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-425049348_22 at /127.0.0.1:44350 [Waiting for operation #37] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-425049348_22 at /127.0.0.1:37850 [Receiving block BP-1980882744-172.17.0.2-1731422596004:blk_1073741899_1077] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-425049348_22 at /127.0.0.1:49842 [Receiving block BP-1980882744-172.17.0.2-1731422596004:blk_1073741899_1077] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-425049348_22 at /127.0.0.1:55000 [Receiving block BP-1980882744-172.17.0.2-1731422596004:blk_1073741899_1077] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-20-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1125 (was 1051) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=260 (was 260), ProcessCount=11 (was 11), AvailableMemoryMB=8364 (was 8418) 2024-11-12T14:43:39,247 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1125 is superior to 1024 2024-11-12T14:43:39,258 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsAfterAbortingFlush Thread=420, OpenFileDescriptor=1125, MaxFileDescriptor=1048576, SystemLoadAverage=260, ProcessCount=11, AvailableMemoryMB=8362 2024-11-12T14:43:39,258 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1125 is superior to 1024 2024-11-12T14:43:39,271 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-12T14:43:39,273 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-12T14:43:39,274 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-12T14:43:39,276 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-33886242, suffix=, logDir=hdfs://localhost:40559/hbase/WALs/hregion-33886242, archiveDir=hdfs://localhost:40559/hbase/oldWALs, maxLogs=32 2024-11-12T14:43:39,289 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-33886242/hregion-33886242.1731422619277, exclude list is [], retry=0 2024-11-12T14:43:39,292 DEBUG [AsyncFSWAL-22-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37395,DS-7317e4d9-cef6-4b85-a947-106c3c917da9,DISK] 2024-11-12T14:43:39,292 DEBUG [AsyncFSWAL-22-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36175,DS-31e7ba0a-e37c-4fe6-8d2a-a4846cefa408,DISK] 2024-11-12T14:43:39,292 DEBUG [AsyncFSWAL-22-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45975,DS-80b047b6-34d2-41c0-99c3-5cd61ca40075,DISK] 2024-11-12T14:43:39,324 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-33886242/hregion-33886242.1731422619277 2024-11-12T14:43:39,325 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40227:40227),(127.0.0.1/127.0.0.1:36027:36027),(127.0.0.1/127.0.0.1:43643:43643)] 2024-11-12T14:43:39,325 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => fe7f4e56ae897d61a30f8c6fca4d9e6f, NAME => 'testReplayEditsAfterAbortingFlush,,1731422619272.fe7f4e56ae897d61a30f8c6fca4d9e6f.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsAfterAbortingFlush', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40559/hbase 2024-11-12T14:43:39,339 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37395 is added to blk_1073741902_1080 (size=68) 2024-11-12T14:43:39,339 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36175 is added to blk_1073741902_1080 (size=68) 2024-11-12T14:43:39,340 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45975 is added to blk_1073741902_1080 (size=68) 2024-11-12T14:43:39,341 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterAbortingFlush,,1731422619272.fe7f4e56ae897d61a30f8c6fca4d9e6f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T14:43:39,342 INFO [StoreOpener-fe7f4e56ae897d61a30f8c6fca4d9e6f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region fe7f4e56ae897d61a30f8c6fca4d9e6f 2024-11-12T14:43:39,344 INFO [StoreOpener-fe7f4e56ae897d61a30f8c6fca4d9e6f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fe7f4e56ae897d61a30f8c6fca4d9e6f columnFamilyName a 2024-11-12T14:43:39,344 DEBUG [StoreOpener-fe7f4e56ae897d61a30f8c6fca4d9e6f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:43:39,344 INFO [StoreOpener-fe7f4e56ae897d61a30f8c6fca4d9e6f-1 {}] regionserver.HStore(327): Store=fe7f4e56ae897d61a30f8c6fca4d9e6f/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T14:43:39,344 INFO [StoreOpener-fe7f4e56ae897d61a30f8c6fca4d9e6f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region fe7f4e56ae897d61a30f8c6fca4d9e6f 2024-11-12T14:43:39,346 INFO [StoreOpener-fe7f4e56ae897d61a30f8c6fca4d9e6f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fe7f4e56ae897d61a30f8c6fca4d9e6f columnFamilyName b 2024-11-12T14:43:39,346 DEBUG [StoreOpener-fe7f4e56ae897d61a30f8c6fca4d9e6f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:43:39,346 INFO [StoreOpener-fe7f4e56ae897d61a30f8c6fca4d9e6f-1 {}] regionserver.HStore(327): Store=fe7f4e56ae897d61a30f8c6fca4d9e6f/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T14:43:39,347 INFO [StoreOpener-fe7f4e56ae897d61a30f8c6fca4d9e6f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region fe7f4e56ae897d61a30f8c6fca4d9e6f 2024-11-12T14:43:39,349 INFO [StoreOpener-fe7f4e56ae897d61a30f8c6fca4d9e6f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fe7f4e56ae897d61a30f8c6fca4d9e6f columnFamilyName c 2024-11-12T14:43:39,349 DEBUG [StoreOpener-fe7f4e56ae897d61a30f8c6fca4d9e6f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:43:39,350 INFO [StoreOpener-fe7f4e56ae897d61a30f8c6fca4d9e6f-1 {}] regionserver.HStore(327): Store=fe7f4e56ae897d61a30f8c6fca4d9e6f/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T14:43:39,351 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for fe7f4e56ae897d61a30f8c6fca4d9e6f 2024-11-12T14:43:39,353 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40559/hbase/data/default/testReplayEditsAfterAbortingFlush/fe7f4e56ae897d61a30f8c6fca4d9e6f 2024-11-12T14:43:39,354 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40559/hbase/data/default/testReplayEditsAfterAbortingFlush/fe7f4e56ae897d61a30f8c6fca4d9e6f 2024-11-12T14:43:39,355 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for fe7f4e56ae897d61a30f8c6fca4d9e6f 2024-11-12T14:43:39,355 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for fe7f4e56ae897d61a30f8c6fca4d9e6f 2024-11-12T14:43:39,356 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterAbortingFlush descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-12T14:43:39,357 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for fe7f4e56ae897d61a30f8c6fca4d9e6f 2024-11-12T14:43:39,363 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40559/hbase/data/default/testReplayEditsAfterAbortingFlush/fe7f4e56ae897d61a30f8c6fca4d9e6f/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-12T14:43:39,363 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened fe7f4e56ae897d61a30f8c6fca4d9e6f; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59019120, jitterRate=-0.12054657936096191}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-12T14:43:39,364 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for fe7f4e56ae897d61a30f8c6fca4d9e6f: Writing region info on filesystem at 1731422619341Initializing all the Stores at 1731422619342 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731422619342Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731422619342Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731422619342Cleaning up temporary data from old regions at 1731422619355 (+13 ms)Region opened successfully at 1731422619364 (+9 ms) 2024-11-12T14:43:39,364 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing fe7f4e56ae897d61a30f8c6fca4d9e6f, disabling compactions & flushes 2024-11-12T14:43:39,364 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterAbortingFlush,,1731422619272.fe7f4e56ae897d61a30f8c6fca4d9e6f. 2024-11-12T14:43:39,364 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterAbortingFlush,,1731422619272.fe7f4e56ae897d61a30f8c6fca4d9e6f. 2024-11-12T14:43:39,365 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterAbortingFlush,,1731422619272.fe7f4e56ae897d61a30f8c6fca4d9e6f. after waiting 0 ms 2024-11-12T14:43:39,365 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterAbortingFlush,,1731422619272.fe7f4e56ae897d61a30f8c6fca4d9e6f. 2024-11-12T14:43:39,365 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenViaHRegion 2024-11-12T14:43:39,365 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenViaHRegion Metrics about Tables on a single HBase RegionServer 2024-11-12T14:43:39,365 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsAfterAbortingFlush,,1731422619272.fe7f4e56ae897d61a30f8c6fca4d9e6f. 2024-11-12T14:43:39,365 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for fe7f4e56ae897d61a30f8c6fca4d9e6f: Waiting for close lock at 1731422619364Disabling compacts and flushes for region at 1731422619364Disabling writes for close at 1731422619365 (+1 ms)Writing region close event to WAL at 1731422619365Closed at 1731422619365 2024-11-12T14:43:39,367 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testSequentialEditLogSeqNum 2024-11-12T14:43:39,367 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testSequentialEditLogSeqNum Metrics about Tables on a single HBase RegionServer 2024-11-12T14:43:39,368 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36175 is added to blk_1073741901_1079 (size=175) 2024-11-12T14:43:39,369 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45975 is added to blk_1073741901_1079 (size=175) 2024-11-12T14:43:39,369 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37395 is added to blk_1073741901_1079 (size=175) 2024-11-12T14:43:39,369 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterRegionMovedWithMultiCF 2024-11-12T14:43:39,370 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterRegionMovedWithMultiCF Metrics about Tables on a single HBase RegionServer 2024-11-12T14:43:39,371 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRegionMadeOfBulkLoadedFilesOnly 2024-11-12T14:43:39,371 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testRegionMadeOfBulkLoadedFilesOnly Metrics about Tables on a single HBase RegionServer 2024-11-12T14:43:39,372 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterAbortingFlush 2024-11-12T14:43:39,373 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterAbortingFlush Metrics about Tables on a single HBase RegionServer 2024-11-12T14:43:39,374 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterAbortingFlush 2024-11-12T14:43:39,380 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-12T14:43:39,380 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-33886242:(num 1731422619277) 2024-11-12T14:43:39,380 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-12T14:43:39,385 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:40559/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731422619271, archiveDir=hdfs://localhost:40559/hbase/oldWALs, maxLogs=32 2024-11-12T14:43:39,401 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731422619271/wal.1731422619385, exclude list is [], retry=0 2024-11-12T14:43:39,404 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36175,DS-31e7ba0a-e37c-4fe6-8d2a-a4846cefa408,DISK] 2024-11-12T14:43:39,404 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45975,DS-80b047b6-34d2-41c0-99c3-5cd61ca40075,DISK] 2024-11-12T14:43:39,405 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37395,DS-7317e4d9-cef6-4b85-a947-106c3c917da9,DISK] 2024-11-12T14:43:39,441 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731422619271/wal.1731422619385 2024-11-12T14:43:39,441 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36027:36027),(127.0.0.1/127.0.0.1:43643:43643),(127.0.0.1/127.0.0.1:40227:40227)] 2024-11-12T14:43:39,533 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => fe7f4e56ae897d61a30f8c6fca4d9e6f, NAME => 'testReplayEditsAfterAbortingFlush,,1731422619272.fe7f4e56ae897d61a30f8c6fca4d9e6f.', STARTKEY => '', ENDKEY => ''} 2024-11-12T14:43:39,536 DEBUG [Time-limited test {}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterAbortingFlush fe7f4e56ae897d61a30f8c6fca4d9e6f 2024-11-12T14:43:39,536 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterAbortingFlush,,1731422619272.fe7f4e56ae897d61a30f8c6fca4d9e6f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T14:43:39,536 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for fe7f4e56ae897d61a30f8c6fca4d9e6f 2024-11-12T14:43:39,536 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for fe7f4e56ae897d61a30f8c6fca4d9e6f 2024-11-12T14:43:39,539 INFO [StoreOpener-fe7f4e56ae897d61a30f8c6fca4d9e6f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region fe7f4e56ae897d61a30f8c6fca4d9e6f 2024-11-12T14:43:39,540 INFO [StoreOpener-fe7f4e56ae897d61a30f8c6fca4d9e6f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fe7f4e56ae897d61a30f8c6fca4d9e6f columnFamilyName a 2024-11-12T14:43:39,540 DEBUG [StoreOpener-fe7f4e56ae897d61a30f8c6fca4d9e6f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:43:39,541 INFO [StoreOpener-fe7f4e56ae897d61a30f8c6fca4d9e6f-1 {}] regionserver.HStore(327): Store=fe7f4e56ae897d61a30f8c6fca4d9e6f/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T14:43:39,541 INFO [StoreOpener-fe7f4e56ae897d61a30f8c6fca4d9e6f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region fe7f4e56ae897d61a30f8c6fca4d9e6f 2024-11-12T14:43:39,542 INFO [StoreOpener-fe7f4e56ae897d61a30f8c6fca4d9e6f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fe7f4e56ae897d61a30f8c6fca4d9e6f columnFamilyName b 2024-11-12T14:43:39,543 DEBUG [StoreOpener-fe7f4e56ae897d61a30f8c6fca4d9e6f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:43:39,543 INFO [StoreOpener-fe7f4e56ae897d61a30f8c6fca4d9e6f-1 {}] regionserver.HStore(327): Store=fe7f4e56ae897d61a30f8c6fca4d9e6f/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T14:43:39,544 INFO [StoreOpener-fe7f4e56ae897d61a30f8c6fca4d9e6f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region fe7f4e56ae897d61a30f8c6fca4d9e6f 2024-11-12T14:43:39,545 INFO [StoreOpener-fe7f4e56ae897d61a30f8c6fca4d9e6f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fe7f4e56ae897d61a30f8c6fca4d9e6f columnFamilyName c 2024-11-12T14:43:39,545 DEBUG [StoreOpener-fe7f4e56ae897d61a30f8c6fca4d9e6f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:43:39,545 INFO [StoreOpener-fe7f4e56ae897d61a30f8c6fca4d9e6f-1 {}] regionserver.HStore(327): Store=fe7f4e56ae897d61a30f8c6fca4d9e6f/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T14:43:39,546 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for fe7f4e56ae897d61a30f8c6fca4d9e6f 2024-11-12T14:43:39,547 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40559/hbase/data/default/testReplayEditsAfterAbortingFlush/fe7f4e56ae897d61a30f8c6fca4d9e6f 2024-11-12T14:43:39,548 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40559/hbase/data/default/testReplayEditsAfterAbortingFlush/fe7f4e56ae897d61a30f8c6fca4d9e6f 2024-11-12T14:43:39,549 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for fe7f4e56ae897d61a30f8c6fca4d9e6f 2024-11-12T14:43:39,549 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for fe7f4e56ae897d61a30f8c6fca4d9e6f 2024-11-12T14:43:39,549 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterAbortingFlush descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-12T14:43:39,550 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for fe7f4e56ae897d61a30f8c6fca4d9e6f 2024-11-12T14:43:39,551 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened fe7f4e56ae897d61a30f8c6fca4d9e6f; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69845227, jitterRate=0.040774986147880554}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-12T14:43:39,551 DEBUG [Time-limited test {}] regionserver.HRegion(1122): Running coprocessor post-open hooks for fe7f4e56ae897d61a30f8c6fca4d9e6f 2024-11-12T14:43:39,551 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for fe7f4e56ae897d61a30f8c6fca4d9e6f: Running coprocessor pre-open hook at 1731422619537Writing region info on filesystem at 1731422619537Initializing all the Stores at 1731422619538 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731422619538Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731422619538Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731422619538Cleaning up temporary data from old regions at 1731422619549 (+11 ms)Running coprocessor post-open hooks at 1731422619551 (+2 ms)Region opened successfully at 1731422619551 2024-11-12T14:43:39,565 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing fe7f4e56ae897d61a30f8c6fca4d9e6f 3/3 column families, dataSize=590 B heapSize=2.08 KB 2024-11-12T14:43:39,567 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for fe7f4e56ae897d61a30f8c6fca4d9e6f/a, retrying num=0 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:43:40,568 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for fe7f4e56ae897d61a30f8c6fca4d9e6f/a, retrying num=1 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:43:40,721 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-12T14:43:41,569 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for fe7f4e56ae897d61a30f8c6fca4d9e6f/a, retrying num=2 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:43:42,570 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for fe7f4e56ae897d61a30f8c6fca4d9e6f/a, retrying num=3 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:43:43,571 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for fe7f4e56ae897d61a30f8c6fca4d9e6f/a, retrying num=4 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:43:44,571 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for fe7f4e56ae897d61a30f8c6fca4d9e6f/a, retrying num=5 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:43:45,573 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for fe7f4e56ae897d61a30f8c6fca4d9e6f/a, retrying num=6 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:43:46,574 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for fe7f4e56ae897d61a30f8c6fca4d9e6f/a, retrying num=7 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:43:47,576 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for fe7f4e56ae897d61a30f8c6fca4d9e6f/a, retrying num=8 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:43:48,209 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-12T14:43:48,577 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for fe7f4e56ae897d61a30f8c6fca4d9e6f/a, retrying num=9 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:43:48,578 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for fe7f4e56ae897d61a30f8c6fca4d9e6f: 2024-11-12T14:43:48,578 INFO [Time-limited test {}] wal.AbstractTestWALReplay(671): Expected simulated exception when flushing region, region: testReplayEditsAfterAbortingFlush,,1731422619272.fe7f4e56ae897d61a30f8c6fca4d9e6f. 2024-11-12T14:43:48,596 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for fe7f4e56ae897d61a30f8c6fca4d9e6f: 2024-11-12T14:43:48,596 INFO [Time-limited test {}] wal.AbstractTestWALReplay(691): Expected exception when flushing region because server is stopped,Aborting flush because server is aborted... 2024-11-12T14:43:48,596 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing fe7f4e56ae897d61a30f8c6fca4d9e6f, disabling compactions & flushes 2024-11-12T14:43:48,597 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterAbortingFlush,,1731422619272.fe7f4e56ae897d61a30f8c6fca4d9e6f. 2024-11-12T14:43:48,597 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterAbortingFlush,,1731422619272.fe7f4e56ae897d61a30f8c6fca4d9e6f. 2024-11-12T14:43:48,597 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterAbortingFlush,,1731422619272.fe7f4e56ae897d61a30f8c6fca4d9e6f. after waiting 0 ms 2024-11-12T14:43:48,597 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterAbortingFlush,,1731422619272.fe7f4e56ae897d61a30f8c6fca4d9e6f. 2024-11-12T14:43:48,597 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 1190 in region testReplayEditsAfterAbortingFlush,,1731422619272.fe7f4e56ae897d61a30f8c6fca4d9e6f. 2024-11-12T14:43:48,597 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsAfterAbortingFlush,,1731422619272.fe7f4e56ae897d61a30f8c6fca4d9e6f. 2024-11-12T14:43:48,597 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for fe7f4e56ae897d61a30f8c6fca4d9e6f: Waiting for close lock at 1731422628596Running coprocessor pre-close hooks at 1731422628596Disabling compacts and flushes for region at 1731422628596Disabling writes for close at 1731422628597 (+1 ms)Writing region close event to WAL at 1731422628597Running coprocessor post-close hooks at 1731422628597Closed at 1731422628597 2024-11-12T14:43:48,606 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731422619271/wal.1731422619385 not finished, retry = 0 2024-11-12T14:43:48,606 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36175 is added to blk_1073741903_1081 (size=4236) 2024-11-12T14:43:48,606 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45975 is added to blk_1073741903_1081 (size=4236) 2024-11-12T14:43:48,607 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37395 is added to blk_1073741903_1081 (size=4236) 2024-11-12T14:43:48,731 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:40559/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731422619271/wal.1731422619385, size=4.1 K (4236bytes) 2024-11-12T14:43:48,731 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:40559/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731422619271/wal.1731422619385 2024-11-12T14:43:48,731 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:40559/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731422619271/wal.1731422619385 after 0ms 2024-11-12T14:43:48,765 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-12T14:43:48,765 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:40559/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731422619271/wal.1731422619385 took 35ms 2024-11-12T14:43:48,768 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:40559/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731422619271/wal.1731422619385 so closing down 2024-11-12T14:43:48,768 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-12T14:43:48,769 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-wal.1731422619385.temp 2024-11-12T14:43:48,804 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:40559/hbase/data/default/testReplayEditsAfterAbortingFlush/fe7f4e56ae897d61a30f8c6fca4d9e6f/recovered.edits/0000000000000000004-wal.1731422619385.temp 2024-11-12T14:43:48,805 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-12T14:43:48,814 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36175 is added to blk_1073741904_1082 (size=3245) 2024-11-12T14:43:48,815 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37395 is added to blk_1073741904_1082 (size=3245) 2024-11-12T14:43:48,815 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45975 is added to blk_1073741904_1082 (size=3245) 2024-11-12T14:43:48,815 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:40559/hbase/data/default/testReplayEditsAfterAbortingFlush/fe7f4e56ae897d61a30f8c6fca4d9e6f/recovered.edits/0000000000000000004-wal.1731422619385.temp (wrote 20 edits, skipped 0 edits in 0 ms) 2024-11-12T14:43:48,817 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:40559/hbase/data/default/testReplayEditsAfterAbortingFlush/fe7f4e56ae897d61a30f8c6fca4d9e6f/recovered.edits/0000000000000000004-wal.1731422619385.temp to hdfs://localhost:40559/hbase/data/default/testReplayEditsAfterAbortingFlush/fe7f4e56ae897d61a30f8c6fca4d9e6f/recovered.edits/0000000000000000026 2024-11-12T14:43:48,817 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 23 edits across 1 Regions in 51 ms; skipped=3; WAL=hdfs://localhost:40559/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731422619271/wal.1731422619385, size=4.1 K, length=4236, corrupted=false, cancelled=false 2024-11-12T14:43:48,817 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:40559/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731422619271/wal.1731422619385, journal: Splitting hdfs://localhost:40559/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731422619271/wal.1731422619385, size=4.1 K (4236bytes) at 1731422628731Finishing writing output for hdfs://localhost:40559/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731422619271/wal.1731422619385 so closing down at 1731422628768 (+37 ms)Creating recovered edits writer path=hdfs://localhost:40559/hbase/data/default/testReplayEditsAfterAbortingFlush/fe7f4e56ae897d61a30f8c6fca4d9e6f/recovered.edits/0000000000000000004-wal.1731422619385.temp at 1731422628804 (+36 ms)3 split writer threads finished at 1731422628805 (+1 ms)Closed recovered edits writer path=hdfs://localhost:40559/hbase/data/default/testReplayEditsAfterAbortingFlush/fe7f4e56ae897d61a30f8c6fca4d9e6f/recovered.edits/0000000000000000004-wal.1731422619385.temp (wrote 20 edits, skipped 0 edits in 0 ms) at 1731422628815 (+10 ms)Rename recovered edits hdfs://localhost:40559/hbase/data/default/testReplayEditsAfterAbortingFlush/fe7f4e56ae897d61a30f8c6fca4d9e6f/recovered.edits/0000000000000000004-wal.1731422619385.temp to hdfs://localhost:40559/hbase/data/default/testReplayEditsAfterAbortingFlush/fe7f4e56ae897d61a30f8c6fca4d9e6f/recovered.edits/0000000000000000026 at 1731422628817 (+2 ms)Processed 23 edits across 1 Regions in 51 ms; skipped=3; WAL=hdfs://localhost:40559/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731422619271/wal.1731422619385, size=4.1 K, length=4236, corrupted=false, cancelled=false at 1731422628817 2024-11-12T14:43:48,819 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:40559/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731422619271/wal.1731422619385 to hdfs://localhost:40559/hbase/oldWALs/wal.1731422619385 2024-11-12T14:43:48,820 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:40559/hbase/data/default/testReplayEditsAfterAbortingFlush/fe7f4e56ae897d61a30f8c6fca4d9e6f/recovered.edits/0000000000000000026 2024-11-12T14:43:48,820 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-12T14:43:48,822 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:40559/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731422619271, archiveDir=hdfs://localhost:40559/hbase/oldWALs, maxLogs=32 2024-11-12T14:43:48,834 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731422619271/wal.1731422628822, exclude list is [], retry=0 2024-11-12T14:43:48,837 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45975,DS-80b047b6-34d2-41c0-99c3-5cd61ca40075,DISK] 2024-11-12T14:43:48,837 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36175,DS-31e7ba0a-e37c-4fe6-8d2a-a4846cefa408,DISK] 2024-11-12T14:43:48,837 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37395,DS-7317e4d9-cef6-4b85-a947-106c3c917da9,DISK] 2024-11-12T14:43:48,866 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731422619271/wal.1731422628822 2024-11-12T14:43:48,866 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43643:43643),(127.0.0.1/127.0.0.1:36027:36027),(127.0.0.1/127.0.0.1:40227:40227)] 2024-11-12T14:43:48,867 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => fe7f4e56ae897d61a30f8c6fca4d9e6f, NAME => 'testReplayEditsAfterAbortingFlush,,1731422619272.fe7f4e56ae897d61a30f8c6fca4d9e6f.', STARTKEY => '', ENDKEY => ''} 2024-11-12T14:43:48,867 DEBUG [Time-limited test {}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterAbortingFlush fe7f4e56ae897d61a30f8c6fca4d9e6f 2024-11-12T14:43:48,867 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterAbortingFlush,,1731422619272.fe7f4e56ae897d61a30f8c6fca4d9e6f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T14:43:48,868 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for fe7f4e56ae897d61a30f8c6fca4d9e6f 2024-11-12T14:43:48,868 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for fe7f4e56ae897d61a30f8c6fca4d9e6f 2024-11-12T14:43:48,870 INFO [StoreOpener-fe7f4e56ae897d61a30f8c6fca4d9e6f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region fe7f4e56ae897d61a30f8c6fca4d9e6f 2024-11-12T14:43:48,871 INFO [StoreOpener-fe7f4e56ae897d61a30f8c6fca4d9e6f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fe7f4e56ae897d61a30f8c6fca4d9e6f columnFamilyName a 2024-11-12T14:43:48,871 DEBUG [StoreOpener-fe7f4e56ae897d61a30f8c6fca4d9e6f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:43:48,873 INFO [StoreOpener-fe7f4e56ae897d61a30f8c6fca4d9e6f-1 {}] regionserver.HStore(327): Store=fe7f4e56ae897d61a30f8c6fca4d9e6f/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T14:43:48,873 INFO [StoreOpener-fe7f4e56ae897d61a30f8c6fca4d9e6f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region fe7f4e56ae897d61a30f8c6fca4d9e6f 2024-11-12T14:43:48,874 INFO [StoreOpener-fe7f4e56ae897d61a30f8c6fca4d9e6f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fe7f4e56ae897d61a30f8c6fca4d9e6f columnFamilyName b 2024-11-12T14:43:48,874 DEBUG [StoreOpener-fe7f4e56ae897d61a30f8c6fca4d9e6f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:43:48,875 INFO [StoreOpener-fe7f4e56ae897d61a30f8c6fca4d9e6f-1 {}] regionserver.HStore(327): Store=fe7f4e56ae897d61a30f8c6fca4d9e6f/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T14:43:48,875 INFO [StoreOpener-fe7f4e56ae897d61a30f8c6fca4d9e6f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region fe7f4e56ae897d61a30f8c6fca4d9e6f 2024-11-12T14:43:48,876 INFO [StoreOpener-fe7f4e56ae897d61a30f8c6fca4d9e6f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fe7f4e56ae897d61a30f8c6fca4d9e6f columnFamilyName c 2024-11-12T14:43:48,877 DEBUG [StoreOpener-fe7f4e56ae897d61a30f8c6fca4d9e6f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:43:48,877 INFO [StoreOpener-fe7f4e56ae897d61a30f8c6fca4d9e6f-1 {}] regionserver.HStore(327): Store=fe7f4e56ae897d61a30f8c6fca4d9e6f/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T14:43:48,878 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for fe7f4e56ae897d61a30f8c6fca4d9e6f 2024-11-12T14:43:48,878 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40559/hbase/data/default/testReplayEditsAfterAbortingFlush/fe7f4e56ae897d61a30f8c6fca4d9e6f 2024-11-12T14:43:48,880 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:40559/hbase/data/default/testReplayEditsAfterAbortingFlush/fe7f4e56ae897d61a30f8c6fca4d9e6f 2024-11-12T14:43:48,881 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:40559/hbase/data/default/testReplayEditsAfterAbortingFlush/fe7f4e56ae897d61a30f8c6fca4d9e6f/recovered.edits/0000000000000000026 2024-11-12T14:43:48,922 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-12T14:43:48,924 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 20, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=26, path=hdfs://localhost:40559/hbase/data/default/testReplayEditsAfterAbortingFlush/fe7f4e56ae897d61a30f8c6fca4d9e6f/recovered.edits/0000000000000000026 2024-11-12T14:43:48,925 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing fe7f4e56ae897d61a30f8c6fca4d9e6f 3/3 column families, dataSize=1.16 KB heapSize=3.41 KB 2024-11-12T14:43:48,947 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40559/hbase/data/default/testReplayEditsAfterAbortingFlush/fe7f4e56ae897d61a30f8c6fca4d9e6f/.tmp/a/eea858a767084e17b34f296808f18e2b is 64, key is testReplayEditsAfterAbortingFlush12/a:q/1731422628584/Put/seqid=0 2024-11-12T14:43:48,953 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37395 is added to blk_1073741906_1084 (size=5523) 2024-11-12T14:43:48,953 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45975 is added to blk_1073741906_1084 (size=5523) 2024-11-12T14:43:48,954 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36175 is added to blk_1073741906_1084 (size=5523) 2024-11-12T14:43:48,955 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=416 B at sequenceid=26 (bloomFilter=true), to=hdfs://localhost:40559/hbase/data/default/testReplayEditsAfterAbortingFlush/fe7f4e56ae897d61a30f8c6fca4d9e6f/.tmp/a/eea858a767084e17b34f296808f18e2b 2024-11-12T14:43:48,987 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40559/hbase/data/default/testReplayEditsAfterAbortingFlush/fe7f4e56ae897d61a30f8c6fca4d9e6f/.tmp/b/770f6e9df2ae4af1b7743136f0ad8964 is 64, key is testReplayEditsAfterAbortingFlush10/b:q/1731422628579/Put/seqid=0 2024-11-12T14:43:48,994 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37395 is added to blk_1073741907_1085 (size=5524) 2024-11-12T14:43:48,994 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36175 is added to blk_1073741907_1085 (size=5524) 2024-11-12T14:43:48,994 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45975 is added to blk_1073741907_1085 (size=5524) 2024-11-12T14:43:48,998 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=417 B at sequenceid=26 (bloomFilter=true), to=hdfs://localhost:40559/hbase/data/default/testReplayEditsAfterAbortingFlush/fe7f4e56ae897d61a30f8c6fca4d9e6f/.tmp/b/770f6e9df2ae4af1b7743136f0ad8964 2024-11-12T14:43:49,028 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40559/hbase/data/default/testReplayEditsAfterAbortingFlush/fe7f4e56ae897d61a30f8c6fca4d9e6f/.tmp/c/b0b17fbce4404a71b8660a293c4fcdc5 is 64, key is testReplayEditsAfterAbortingFlush11/c:q/1731422628582/Put/seqid=0 2024-11-12T14:43:49,039 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45975 is added to blk_1073741908_1086 (size=5457) 2024-11-12T14:43:49,040 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37395 is added to blk_1073741908_1086 (size=5457) 2024-11-12T14:43:49,040 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36175 is added to blk_1073741908_1086 (size=5457) 2024-11-12T14:43:49,040 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=357 B at sequenceid=26 (bloomFilter=true), to=hdfs://localhost:40559/hbase/data/default/testReplayEditsAfterAbortingFlush/fe7f4e56ae897d61a30f8c6fca4d9e6f/.tmp/c/b0b17fbce4404a71b8660a293c4fcdc5 2024-11-12T14:43:49,049 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40559/hbase/data/default/testReplayEditsAfterAbortingFlush/fe7f4e56ae897d61a30f8c6fca4d9e6f/.tmp/a/eea858a767084e17b34f296808f18e2b as hdfs://localhost:40559/hbase/data/default/testReplayEditsAfterAbortingFlush/fe7f4e56ae897d61a30f8c6fca4d9e6f/a/eea858a767084e17b34f296808f18e2b 2024-11-12T14:43:49,056 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40559/hbase/data/default/testReplayEditsAfterAbortingFlush/fe7f4e56ae897d61a30f8c6fca4d9e6f/a/eea858a767084e17b34f296808f18e2b, entries=7, sequenceid=26, filesize=5.4 K 2024-11-12T14:43:49,058 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40559/hbase/data/default/testReplayEditsAfterAbortingFlush/fe7f4e56ae897d61a30f8c6fca4d9e6f/.tmp/b/770f6e9df2ae4af1b7743136f0ad8964 as hdfs://localhost:40559/hbase/data/default/testReplayEditsAfterAbortingFlush/fe7f4e56ae897d61a30f8c6fca4d9e6f/b/770f6e9df2ae4af1b7743136f0ad8964 2024-11-12T14:43:49,065 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40559/hbase/data/default/testReplayEditsAfterAbortingFlush/fe7f4e56ae897d61a30f8c6fca4d9e6f/b/770f6e9df2ae4af1b7743136f0ad8964, entries=7, sequenceid=26, filesize=5.4 K 2024-11-12T14:43:49,069 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40559/hbase/data/default/testReplayEditsAfterAbortingFlush/fe7f4e56ae897d61a30f8c6fca4d9e6f/.tmp/c/b0b17fbce4404a71b8660a293c4fcdc5 as hdfs://localhost:40559/hbase/data/default/testReplayEditsAfterAbortingFlush/fe7f4e56ae897d61a30f8c6fca4d9e6f/c/b0b17fbce4404a71b8660a293c4fcdc5 2024-11-12T14:43:49,077 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40559/hbase/data/default/testReplayEditsAfterAbortingFlush/fe7f4e56ae897d61a30f8c6fca4d9e6f/c/b0b17fbce4404a71b8660a293c4fcdc5, entries=6, sequenceid=26, filesize=5.3 K 2024-11-12T14:43:49,077 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~1.16 KB/1190, heapSize ~3.36 KB/3440, currentSize=0 B/0 for fe7f4e56ae897d61a30f8c6fca4d9e6f in 152ms, sequenceid=26, compaction requested=false; wal=null 2024-11-12T14:43:49,078 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:40559/hbase/data/default/testReplayEditsAfterAbortingFlush/fe7f4e56ae897d61a30f8c6fca4d9e6f/recovered.edits/0000000000000000026 2024-11-12T14:43:49,079 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for fe7f4e56ae897d61a30f8c6fca4d9e6f 2024-11-12T14:43:49,079 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for fe7f4e56ae897d61a30f8c6fca4d9e6f 2024-11-12T14:43:49,080 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterAbortingFlush descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-12T14:43:49,082 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for fe7f4e56ae897d61a30f8c6fca4d9e6f 2024-11-12T14:43:49,085 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40559/hbase/data/default/testReplayEditsAfterAbortingFlush/fe7f4e56ae897d61a30f8c6fca4d9e6f/recovered.edits/26.seqid, newMaxSeqId=26, maxSeqId=1 2024-11-12T14:43:49,086 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened fe7f4e56ae897d61a30f8c6fca4d9e6f; next sequenceid=27; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68441436, jitterRate=0.01985687017440796}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-12T14:43:49,087 DEBUG [Time-limited test {}] regionserver.HRegion(1122): Running coprocessor post-open hooks for fe7f4e56ae897d61a30f8c6fca4d9e6f 2024-11-12T14:43:49,087 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for fe7f4e56ae897d61a30f8c6fca4d9e6f: Running coprocessor pre-open hook at 1731422628868Writing region info on filesystem at 1731422628868Initializing all the Stores at 1731422628869 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731422628869Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731422628869Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731422628869Obtaining lock to block concurrent updates at 1731422628925 (+56 ms)Preparing flush snapshotting stores in fe7f4e56ae897d61a30f8c6fca4d9e6f at 1731422628925Finished memstore snapshotting testReplayEditsAfterAbortingFlush,,1731422619272.fe7f4e56ae897d61a30f8c6fca4d9e6f., syncing WAL and waiting on mvcc, flushsize=dataSize=1190, getHeapSize=3440, getOffHeapSize=0, getCellsCount=20 at 1731422628925Flushing stores of testReplayEditsAfterAbortingFlush,,1731422619272.fe7f4e56ae897d61a30f8c6fca4d9e6f. at 1731422628925Flushing fe7f4e56ae897d61a30f8c6fca4d9e6f/a: creating writer at 1731422628926 (+1 ms)Flushing fe7f4e56ae897d61a30f8c6fca4d9e6f/a: appending metadata at 1731422628946 (+20 ms)Flushing fe7f4e56ae897d61a30f8c6fca4d9e6f/a: closing flushed file at 1731422628946Flushing fe7f4e56ae897d61a30f8c6fca4d9e6f/b: creating writer at 1731422628963 (+17 ms)Flushing fe7f4e56ae897d61a30f8c6fca4d9e6f/b: appending metadata at 1731422628986 (+23 ms)Flushing fe7f4e56ae897d61a30f8c6fca4d9e6f/b: closing flushed file at 1731422628986Flushing fe7f4e56ae897d61a30f8c6fca4d9e6f/c: creating writer at 1731422629005 (+19 ms)Flushing fe7f4e56ae897d61a30f8c6fca4d9e6f/c: appending metadata at 1731422629027 (+22 ms)Flushing fe7f4e56ae897d61a30f8c6fca4d9e6f/c: closing flushed file at 1731422629027Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@287b140a: reopening flushed file at 1731422629048 (+21 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7ce78d2e: reopening flushed file at 1731422629057 (+9 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@468f54dd: reopening flushed file at 1731422629065 (+8 ms)Finished flush of dataSize ~1.16 KB/1190, heapSize ~3.36 KB/3440, currentSize=0 B/0 for fe7f4e56ae897d61a30f8c6fca4d9e6f in 152ms, sequenceid=26, compaction requested=false; wal=null at 1731422629077 (+12 ms)Cleaning up temporary data from old regions at 1731422629079 (+2 ms)Running coprocessor post-open hooks at 1731422629087 (+8 ms)Region opened successfully at 1731422629087 2024-11-12T14:43:49,117 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsAfterAbortingFlush Thread=420 (was 420), OpenFileDescriptor=1180 (was 1125) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=251 (was 260), ProcessCount=11 (was 11), AvailableMemoryMB=8285 (was 8362) 2024-11-12T14:43:49,117 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1180 is superior to 1024 2024-11-12T14:43:49,136 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testDatalossWhenInputError Thread=420, OpenFileDescriptor=1180, MaxFileDescriptor=1048576, SystemLoadAverage=251, ProcessCount=11, AvailableMemoryMB=8276 2024-11-12T14:43:49,136 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1180 is superior to 1024 2024-11-12T14:43:49,156 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-12T14:43:49,159 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-12T14:43:49,159 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-12T14:43:49,162 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-22632915, suffix=, logDir=hdfs://localhost:40559/hbase/WALs/hregion-22632915, archiveDir=hdfs://localhost:40559/hbase/oldWALs, maxLogs=32 2024-11-12T14:43:49,175 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-22632915/hregion-22632915.1731422629162, exclude list is [], retry=0 2024-11-12T14:43:49,178 DEBUG [AsyncFSWAL-24-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45975,DS-80b047b6-34d2-41c0-99c3-5cd61ca40075,DISK] 2024-11-12T14:43:49,178 DEBUG [AsyncFSWAL-24-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37395,DS-7317e4d9-cef6-4b85-a947-106c3c917da9,DISK] 2024-11-12T14:43:49,178 DEBUG [AsyncFSWAL-24-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36175,DS-31e7ba0a-e37c-4fe6-8d2a-a4846cefa408,DISK] 2024-11-12T14:43:49,216 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-22632915/hregion-22632915.1731422629162 2024-11-12T14:43:49,217 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40227:40227),(127.0.0.1/127.0.0.1:43643:43643),(127.0.0.1/127.0.0.1:36027:36027)] 2024-11-12T14:43:49,217 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 7b056dc9994407314702fef1d77e82c2, NAME => 'testDatalossWhenInputError,,1731422629157.7b056dc9994407314702fef1d77e82c2.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testDatalossWhenInputError', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40559/hbase 2024-11-12T14:43:49,238 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36175 is added to blk_1073741910_1088 (size=61) 2024-11-12T14:43:49,238 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45975 is added to blk_1073741910_1088 (size=61) 2024-11-12T14:43:49,239 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37395 is added to blk_1073741910_1088 (size=61) 2024-11-12T14:43:49,240 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1731422629157.7b056dc9994407314702fef1d77e82c2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T14:43:49,241 INFO [StoreOpener-7b056dc9994407314702fef1d77e82c2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 7b056dc9994407314702fef1d77e82c2 2024-11-12T14:43:49,242 INFO [StoreOpener-7b056dc9994407314702fef1d77e82c2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7b056dc9994407314702fef1d77e82c2 columnFamilyName a 2024-11-12T14:43:49,242 DEBUG [StoreOpener-7b056dc9994407314702fef1d77e82c2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:43:49,243 INFO [StoreOpener-7b056dc9994407314702fef1d77e82c2-1 {}] regionserver.HStore(327): Store=7b056dc9994407314702fef1d77e82c2/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T14:43:49,243 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 7b056dc9994407314702fef1d77e82c2 2024-11-12T14:43:49,244 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40559/hbase/data/default/testDatalossWhenInputError/7b056dc9994407314702fef1d77e82c2 2024-11-12T14:43:49,246 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40559/hbase/data/default/testDatalossWhenInputError/7b056dc9994407314702fef1d77e82c2 2024-11-12T14:43:49,246 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 7b056dc9994407314702fef1d77e82c2 2024-11-12T14:43:49,246 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 7b056dc9994407314702fef1d77e82c2 2024-11-12T14:43:49,248 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 7b056dc9994407314702fef1d77e82c2 2024-11-12T14:43:49,254 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40559/hbase/data/default/testDatalossWhenInputError/7b056dc9994407314702fef1d77e82c2/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-12T14:43:49,254 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 7b056dc9994407314702fef1d77e82c2; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71043329, jitterRate=0.05862809717655182}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-12T14:43:49,255 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 7b056dc9994407314702fef1d77e82c2: Writing region info on filesystem at 1731422629240Initializing all the Stores at 1731422629240Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731422629240Cleaning up temporary data from old regions at 1731422629246 (+6 ms)Region opened successfully at 1731422629255 (+9 ms) 2024-11-12T14:43:49,255 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 7b056dc9994407314702fef1d77e82c2, disabling compactions & flushes 2024-11-12T14:43:49,255 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testDatalossWhenInputError,,1731422629157.7b056dc9994407314702fef1d77e82c2. 2024-11-12T14:43:49,256 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testDatalossWhenInputError,,1731422629157.7b056dc9994407314702fef1d77e82c2. 2024-11-12T14:43:49,256 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testDatalossWhenInputError,,1731422629157.7b056dc9994407314702fef1d77e82c2. after waiting 0 ms 2024-11-12T14:43:49,256 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testDatalossWhenInputError,,1731422629157.7b056dc9994407314702fef1d77e82c2. 2024-11-12T14:43:49,256 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testDatalossWhenInputError,,1731422629157.7b056dc9994407314702fef1d77e82c2. 2024-11-12T14:43:49,256 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 7b056dc9994407314702fef1d77e82c2: Waiting for close lock at 1731422629255Disabling compacts and flushes for region at 1731422629255Disabling writes for close at 1731422629256 (+1 ms)Writing region close event to WAL at 1731422629256Closed at 1731422629256 2024-11-12T14:43:49,261 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45975 is added to blk_1073741909_1087 (size=175) 2024-11-12T14:43:49,261 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36175 is added to blk_1073741909_1087 (size=175) 2024-11-12T14:43:49,261 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37395 is added to blk_1073741909_1087 (size=175) 2024-11-12T14:43:49,264 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-12T14:43:49,265 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-22632915:(num 1731422629162) 2024-11-12T14:43:49,265 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-12T14:43:49,267 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:40559/hbase/WALs/testdatalosswheninputerror-manual,16010,1731422629155, archiveDir=hdfs://localhost:40559/hbase/oldWALs, maxLogs=32 2024-11-12T14:43:49,281 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testdatalosswheninputerror-manual,16010,1731422629155/wal.1731422629268, exclude list is [], retry=0 2024-11-12T14:43:49,284 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36175,DS-31e7ba0a-e37c-4fe6-8d2a-a4846cefa408,DISK] 2024-11-12T14:43:49,284 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37395,DS-7317e4d9-cef6-4b85-a947-106c3c917da9,DISK] 2024-11-12T14:43:49,284 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45975,DS-80b047b6-34d2-41c0-99c3-5cd61ca40075,DISK] 2024-11-12T14:43:49,309 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testdatalosswheninputerror-manual,16010,1731422629155/wal.1731422629268 2024-11-12T14:43:49,309 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36027:36027),(127.0.0.1/127.0.0.1:40227:40227),(127.0.0.1/127.0.0.1:43643:43643)] 2024-11-12T14:43:49,309 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 7b056dc9994407314702fef1d77e82c2, NAME => 'testDatalossWhenInputError,,1731422629157.7b056dc9994407314702fef1d77e82c2.', STARTKEY => '', ENDKEY => ''} 2024-11-12T14:43:49,310 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1731422629157.7b056dc9994407314702fef1d77e82c2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T14:43:49,310 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 7b056dc9994407314702fef1d77e82c2 2024-11-12T14:43:49,310 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 7b056dc9994407314702fef1d77e82c2 2024-11-12T14:43:49,311 INFO [StoreOpener-7b056dc9994407314702fef1d77e82c2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 7b056dc9994407314702fef1d77e82c2 2024-11-12T14:43:49,312 INFO [StoreOpener-7b056dc9994407314702fef1d77e82c2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7b056dc9994407314702fef1d77e82c2 columnFamilyName a 2024-11-12T14:43:49,312 DEBUG [StoreOpener-7b056dc9994407314702fef1d77e82c2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:43:49,314 INFO [StoreOpener-7b056dc9994407314702fef1d77e82c2-1 {}] regionserver.HStore(327): Store=7b056dc9994407314702fef1d77e82c2/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T14:43:49,314 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 7b056dc9994407314702fef1d77e82c2 2024-11-12T14:43:49,315 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40559/hbase/data/default/testDatalossWhenInputError/7b056dc9994407314702fef1d77e82c2 2024-11-12T14:43:49,319 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40559/hbase/data/default/testDatalossWhenInputError/7b056dc9994407314702fef1d77e82c2 2024-11-12T14:43:49,320 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 7b056dc9994407314702fef1d77e82c2 2024-11-12T14:43:49,320 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 7b056dc9994407314702fef1d77e82c2 2024-11-12T14:43:49,322 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 7b056dc9994407314702fef1d77e82c2 2024-11-12T14:43:49,324 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 7b056dc9994407314702fef1d77e82c2; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61528581, jitterRate=-0.08315269649028778}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-12T14:43:49,325 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 7b056dc9994407314702fef1d77e82c2: Writing region info on filesystem at 1731422629310Initializing all the Stores at 1731422629311 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731422629311Cleaning up temporary data from old regions at 1731422629320 (+9 ms)Region opened successfully at 1731422629325 (+5 ms) 2024-11-12T14:43:49,339 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 7b056dc9994407314702fef1d77e82c2, disabling compactions & flushes 2024-11-12T14:43:49,339 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testDatalossWhenInputError,,1731422629157.7b056dc9994407314702fef1d77e82c2. 2024-11-12T14:43:49,339 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testDatalossWhenInputError,,1731422629157.7b056dc9994407314702fef1d77e82c2. 2024-11-12T14:43:49,339 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testDatalossWhenInputError,,1731422629157.7b056dc9994407314702fef1d77e82c2. after waiting 0 ms 2024-11-12T14:43:49,339 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testDatalossWhenInputError,,1731422629157.7b056dc9994407314702fef1d77e82c2. 2024-11-12T14:43:49,339 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 750 in region testDatalossWhenInputError,,1731422629157.7b056dc9994407314702fef1d77e82c2. 2024-11-12T14:43:49,339 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testDatalossWhenInputError,,1731422629157.7b056dc9994407314702fef1d77e82c2. 2024-11-12T14:43:49,340 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 7b056dc9994407314702fef1d77e82c2: Waiting for close lock at 1731422629339Disabling compacts and flushes for region at 1731422629339Disabling writes for close at 1731422629339Writing region close event to WAL at 1731422629339Closed at 1731422629339 2024-11-12T14:43:49,345 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36175 is added to blk_1073741911_1089 (size=1795) 2024-11-12T14:43:49,345 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45975 is added to blk_1073741911_1089 (size=1795) 2024-11-12T14:43:49,345 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37395 is added to blk_1073741911_1089 (size=1795) 2024-11-12T14:43:49,363 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:40559/hbase/WALs/testdatalosswheninputerror-manual,16010,1731422629155/wal.1731422629268, size=1.8 K (1795bytes) 2024-11-12T14:43:49,363 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:40559/hbase/WALs/testdatalosswheninputerror-manual,16010,1731422629155/wal.1731422629268 2024-11-12T14:43:49,364 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:40559/hbase/WALs/testdatalosswheninputerror-manual,16010,1731422629155/wal.1731422629268 after 1ms 2024-11-12T14:43:49,365 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterAbortingFlush 2024-11-12T14:43:49,365 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterAbortingFlush Metrics about Tables on a single HBase RegionServer 2024-11-12T14:43:49,394 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-12T14:43:49,394 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:40559/hbase/WALs/testdatalosswheninputerror-manual,16010,1731422629155/wal.1731422629268 took 31ms 2024-11-12T14:43:49,396 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:40559/hbase/WALs/testdatalosswheninputerror-manual,16010,1731422629155/wal.1731422629268 so closing down 2024-11-12T14:43:49,396 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-12T14:43:49,397 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1731422629268.temp 2024-11-12T14:43:49,450 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:40559/hbase/data/default/testDatalossWhenInputError/7b056dc9994407314702fef1d77e82c2/recovered.edits/0000000000000000003-wal.1731422629268.temp 2024-11-12T14:43:49,451 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-12T14:43:49,461 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45975 is added to blk_1073741912_1090 (size=1795) 2024-11-12T14:43:49,461 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36175 is added to blk_1073741912_1090 (size=1795) 2024-11-12T14:43:49,461 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37395 is added to blk_1073741912_1090 (size=1795) 2024-11-12T14:43:49,462 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:40559/hbase/data/default/testDatalossWhenInputError/7b056dc9994407314702fef1d77e82c2/recovered.edits/0000000000000000003-wal.1731422629268.temp (wrote 10 edits, skipped 0 edits in 0 ms) 2024-11-12T14:43:49,463 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:40559/hbase/data/default/testDatalossWhenInputError/7b056dc9994407314702fef1d77e82c2/recovered.edits/0000000000000000003-wal.1731422629268.temp to hdfs://localhost:40559/hbase/data/default/testDatalossWhenInputError/7b056dc9994407314702fef1d77e82c2/recovered.edits/0000000000000000012 2024-11-12T14:43:49,463 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 10 edits across 1 Regions in 69 ms; skipped=0; WAL=hdfs://localhost:40559/hbase/WALs/testdatalosswheninputerror-manual,16010,1731422629155/wal.1731422629268, size=1.8 K, length=1795, corrupted=false, cancelled=false 2024-11-12T14:43:49,463 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:40559/hbase/WALs/testdatalosswheninputerror-manual,16010,1731422629155/wal.1731422629268, journal: Splitting hdfs://localhost:40559/hbase/WALs/testdatalosswheninputerror-manual,16010,1731422629155/wal.1731422629268, size=1.8 K (1795bytes) at 1731422629363Finishing writing output for hdfs://localhost:40559/hbase/WALs/testdatalosswheninputerror-manual,16010,1731422629155/wal.1731422629268 so closing down at 1731422629396 (+33 ms)Creating recovered edits writer path=hdfs://localhost:40559/hbase/data/default/testDatalossWhenInputError/7b056dc9994407314702fef1d77e82c2/recovered.edits/0000000000000000003-wal.1731422629268.temp at 1731422629450 (+54 ms)3 split writer threads finished at 1731422629451 (+1 ms)Closed recovered edits writer path=hdfs://localhost:40559/hbase/data/default/testDatalossWhenInputError/7b056dc9994407314702fef1d77e82c2/recovered.edits/0000000000000000003-wal.1731422629268.temp (wrote 10 edits, skipped 0 edits in 0 ms) at 1731422629462 (+11 ms)Rename recovered edits hdfs://localhost:40559/hbase/data/default/testDatalossWhenInputError/7b056dc9994407314702fef1d77e82c2/recovered.edits/0000000000000000003-wal.1731422629268.temp to hdfs://localhost:40559/hbase/data/default/testDatalossWhenInputError/7b056dc9994407314702fef1d77e82c2/recovered.edits/0000000000000000012 at 1731422629463 (+1 ms)Processed 10 edits across 1 Regions in 69 ms; skipped=0; WAL=hdfs://localhost:40559/hbase/WALs/testdatalosswheninputerror-manual,16010,1731422629155/wal.1731422629268, size=1.8 K, length=1795, corrupted=false, cancelled=false at 1731422629463 2024-11-12T14:43:49,465 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:40559/hbase/WALs/testdatalosswheninputerror-manual,16010,1731422629155/wal.1731422629268 to hdfs://localhost:40559/hbase/oldWALs/wal.1731422629268 2024-11-12T14:43:49,466 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:40559/hbase/data/default/testDatalossWhenInputError/7b056dc9994407314702fef1d77e82c2/recovered.edits/0000000000000000012 2024-11-12T14:43:49,495 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-12T14:43:49,944 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-12T14:43:49,946 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:40559/hbase/WALs/testdatalosswheninputerror-manual,16010,1731422629155, archiveDir=hdfs://localhost:40559/hbase/oldWALs, maxLogs=32 2024-11-12T14:43:49,963 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testdatalosswheninputerror-manual,16010,1731422629155/wal.1731422629947, exclude list is [], retry=0 2024-11-12T14:43:49,966 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45975,DS-80b047b6-34d2-41c0-99c3-5cd61ca40075,DISK] 2024-11-12T14:43:49,966 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37395,DS-7317e4d9-cef6-4b85-a947-106c3c917da9,DISK] 2024-11-12T14:43:49,967 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36175,DS-31e7ba0a-e37c-4fe6-8d2a-a4846cefa408,DISK] 2024-11-12T14:43:49,997 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testdatalosswheninputerror-manual,16010,1731422629155/wal.1731422629947 2024-11-12T14:43:49,998 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43643:43643),(127.0.0.1/127.0.0.1:40227:40227),(127.0.0.1/127.0.0.1:36027:36027)] 2024-11-12T14:43:49,998 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 7b056dc9994407314702fef1d77e82c2, NAME => 'testDatalossWhenInputError,,1731422629157.7b056dc9994407314702fef1d77e82c2.', STARTKEY => '', ENDKEY => ''} 2024-11-12T14:43:49,998 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1731422629157.7b056dc9994407314702fef1d77e82c2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T14:43:49,998 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 7b056dc9994407314702fef1d77e82c2 2024-11-12T14:43:49,998 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 7b056dc9994407314702fef1d77e82c2 2024-11-12T14:43:50,002 INFO [StoreOpener-7b056dc9994407314702fef1d77e82c2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 7b056dc9994407314702fef1d77e82c2 2024-11-12T14:43:50,004 INFO [StoreOpener-7b056dc9994407314702fef1d77e82c2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7b056dc9994407314702fef1d77e82c2 columnFamilyName a 2024-11-12T14:43:50,004 DEBUG [StoreOpener-7b056dc9994407314702fef1d77e82c2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:43:50,005 INFO [StoreOpener-7b056dc9994407314702fef1d77e82c2-1 {}] regionserver.HStore(327): Store=7b056dc9994407314702fef1d77e82c2/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T14:43:50,005 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 7b056dc9994407314702fef1d77e82c2 2024-11-12T14:43:50,006 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40559/hbase/data/default/testDatalossWhenInputError/7b056dc9994407314702fef1d77e82c2 2024-11-12T14:43:50,008 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:40559/hbase/data/default/testDatalossWhenInputError/7b056dc9994407314702fef1d77e82c2 2024-11-12T14:43:50,009 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:40559/hbase/data/default/testDatalossWhenInputError/7b056dc9994407314702fef1d77e82c2/recovered.edits/0000000000000000012 2024-11-12T14:43:50,038 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-12T14:43:50,039 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 10, skipped 0, firstSequenceIdInLog=3, maxSequenceIdInLog=12, path=hdfs://localhost:40559/hbase/data/default/testDatalossWhenInputError/7b056dc9994407314702fef1d77e82c2/recovered.edits/0000000000000000012 2024-11-12T14:43:50,039 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 7b056dc9994407314702fef1d77e82c2 1/1 column families, dataSize=750 B heapSize=1.73 KB 2024-11-12T14:43:50,059 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40559/hbase/data/default/testDatalossWhenInputError/7b056dc9994407314702fef1d77e82c2/.tmp/a/ffa8ed8966c44067a4d0f705f7dbd408 is 79, key is testDatalossWhenInputError/a:x0/1731422629325/Put/seqid=0 2024-11-12T14:43:50,071 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37395 is added to blk_1073741914_1092 (size=5808) 2024-11-12T14:43:50,072 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45975 is added to blk_1073741914_1092 (size=5808) 2024-11-12T14:43:50,072 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36175 is added to blk_1073741914_1092 (size=5808) 2024-11-12T14:43:50,073 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=750 B at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:40559/hbase/data/default/testDatalossWhenInputError/7b056dc9994407314702fef1d77e82c2/.tmp/a/ffa8ed8966c44067a4d0f705f7dbd408 2024-11-12T14:43:50,086 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40559/hbase/data/default/testDatalossWhenInputError/7b056dc9994407314702fef1d77e82c2/.tmp/a/ffa8ed8966c44067a4d0f705f7dbd408 as hdfs://localhost:40559/hbase/data/default/testDatalossWhenInputError/7b056dc9994407314702fef1d77e82c2/a/ffa8ed8966c44067a4d0f705f7dbd408 2024-11-12T14:43:50,098 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40559/hbase/data/default/testDatalossWhenInputError/7b056dc9994407314702fef1d77e82c2/a/ffa8ed8966c44067a4d0f705f7dbd408, entries=10, sequenceid=12, filesize=5.7 K 2024-11-12T14:43:50,098 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~750 B/750, heapSize ~1.72 KB/1760, currentSize=0 B/0 for 7b056dc9994407314702fef1d77e82c2 in 59ms, sequenceid=12, compaction requested=false; wal=null 2024-11-12T14:43:50,099 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:40559/hbase/data/default/testDatalossWhenInputError/7b056dc9994407314702fef1d77e82c2/recovered.edits/0000000000000000012 2024-11-12T14:43:50,100 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 7b056dc9994407314702fef1d77e82c2 2024-11-12T14:43:50,100 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 7b056dc9994407314702fef1d77e82c2 2024-11-12T14:43:50,104 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 7b056dc9994407314702fef1d77e82c2 2024-11-12T14:43:50,106 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40559/hbase/data/default/testDatalossWhenInputError/7b056dc9994407314702fef1d77e82c2/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=1 2024-11-12T14:43:50,107 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 7b056dc9994407314702fef1d77e82c2; next sequenceid=13; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=58883591, jitterRate=-0.1225661188364029}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-12T14:43:50,108 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 7b056dc9994407314702fef1d77e82c2: Writing region info on filesystem at 1731422629998Initializing all the Stores at 1731422630001 (+3 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731422630001Obtaining lock to block concurrent updates at 1731422630039 (+38 ms)Preparing flush snapshotting stores in 7b056dc9994407314702fef1d77e82c2 at 1731422630039Finished memstore snapshotting testDatalossWhenInputError,,1731422629157.7b056dc9994407314702fef1d77e82c2., syncing WAL and waiting on mvcc, flushsize=dataSize=750, getHeapSize=1760, getOffHeapSize=0, getCellsCount=10 at 1731422630039Flushing stores of testDatalossWhenInputError,,1731422629157.7b056dc9994407314702fef1d77e82c2. at 1731422630039Flushing 7b056dc9994407314702fef1d77e82c2/a: creating writer at 1731422630040 (+1 ms)Flushing 7b056dc9994407314702fef1d77e82c2/a: appending metadata at 1731422630059 (+19 ms)Flushing 7b056dc9994407314702fef1d77e82c2/a: closing flushed file at 1731422630059Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4e93f4ad: reopening flushed file at 1731422630084 (+25 ms)Finished flush of dataSize ~750 B/750, heapSize ~1.72 KB/1760, currentSize=0 B/0 for 7b056dc9994407314702fef1d77e82c2 in 59ms, sequenceid=12, compaction requested=false; wal=null at 1731422630098 (+14 ms)Cleaning up temporary data from old regions at 1731422630100 (+2 ms)Region opened successfully at 1731422630108 (+8 ms) 2024-11-12T14:43:50,111 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 7b056dc9994407314702fef1d77e82c2, NAME => 'testDatalossWhenInputError,,1731422629157.7b056dc9994407314702fef1d77e82c2.', STARTKEY => '', ENDKEY => ''} 2024-11-12T14:43:50,111 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1731422629157.7b056dc9994407314702fef1d77e82c2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T14:43:50,111 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 7b056dc9994407314702fef1d77e82c2 2024-11-12T14:43:50,111 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 7b056dc9994407314702fef1d77e82c2 2024-11-12T14:43:50,112 INFO [StoreOpener-7b056dc9994407314702fef1d77e82c2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 7b056dc9994407314702fef1d77e82c2 2024-11-12T14:43:50,113 INFO [StoreOpener-7b056dc9994407314702fef1d77e82c2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7b056dc9994407314702fef1d77e82c2 columnFamilyName a 2024-11-12T14:43:50,113 DEBUG [StoreOpener-7b056dc9994407314702fef1d77e82c2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:43:50,119 DEBUG [StoreOpener-7b056dc9994407314702fef1d77e82c2-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:40559/hbase/data/default/testDatalossWhenInputError/7b056dc9994407314702fef1d77e82c2/a/ffa8ed8966c44067a4d0f705f7dbd408 2024-11-12T14:43:50,119 INFO [StoreOpener-7b056dc9994407314702fef1d77e82c2-1 {}] regionserver.HStore(327): Store=7b056dc9994407314702fef1d77e82c2/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T14:43:50,119 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 7b056dc9994407314702fef1d77e82c2 2024-11-12T14:43:50,120 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40559/hbase/data/default/testDatalossWhenInputError/7b056dc9994407314702fef1d77e82c2 2024-11-12T14:43:50,121 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40559/hbase/data/default/testDatalossWhenInputError/7b056dc9994407314702fef1d77e82c2 2024-11-12T14:43:50,122 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 7b056dc9994407314702fef1d77e82c2 2024-11-12T14:43:50,122 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 7b056dc9994407314702fef1d77e82c2 2024-11-12T14:43:50,124 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 7b056dc9994407314702fef1d77e82c2 2024-11-12T14:43:50,127 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40559/hbase/data/default/testDatalossWhenInputError/7b056dc9994407314702fef1d77e82c2/recovered.edits/13.seqid, newMaxSeqId=13, maxSeqId=12 2024-11-12T14:43:50,128 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 7b056dc9994407314702fef1d77e82c2; next sequenceid=14; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=58953934, jitterRate=-0.12151792645454407}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-12T14:43:50,129 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 7b056dc9994407314702fef1d77e82c2: Writing region info on filesystem at 1731422630111Initializing all the Stores at 1731422630112 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731422630112Cleaning up temporary data from old regions at 1731422630122 (+10 ms)Region opened successfully at 1731422630129 (+7 ms) 2024-11-12T14:43:50,150 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testDatalossWhenInputError Thread=431 (was 420) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-425049348_22 at /127.0.0.1:60134 [Receiving block BP-1980882744-172.17.0.2-1731422596004:blk_1073741913_1091] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-24-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-425049348_22 at /127.0.0.1:59760 [Receiving block BP-1980882744-172.17.0.2-1731422596004:blk_1073741913_1091] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1980882744-172.17.0.2-1731422596004:blk_1073741913_1091, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-24-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-425049348_22 at /127.0.0.1:59750 [Waiting for operation #9] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1980882744-172.17.0.2-1731422596004:blk_1073741913_1091, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-425049348_22 at /127.0.0.1:55044 [Waiting for operation #16] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-425049348_22 at /127.0.0.1:60058 [Waiting for operation #17] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-425049348_22 at /127.0.0.1:41340 [Receiving block BP-1980882744-172.17.0.2-1731422596004:blk_1073741913_1091] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-24-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1980882744-172.17.0.2-1731422596004:blk_1073741913_1091, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1264 (was 1180) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=251 (was 251), ProcessCount=11 (was 11), AvailableMemoryMB=8189 (was 8276) 2024-11-12T14:43:50,151 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1264 is superior to 1024 2024-11-12T14:43:50,164 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testCompactedBulkLoadedFiles Thread=431, OpenFileDescriptor=1264, MaxFileDescriptor=1048576, SystemLoadAverage=251, ProcessCount=11, AvailableMemoryMB=8186 2024-11-12T14:43:50,164 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1264 is superior to 1024 2024-11-12T14:43:50,180 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-12T14:43:50,182 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-12T14:43:50,183 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-12T14:43:50,186 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-45775324, suffix=, logDir=hdfs://localhost:40559/hbase/WALs/hregion-45775324, archiveDir=hdfs://localhost:40559/hbase/oldWALs, maxLogs=32 2024-11-12T14:43:50,202 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-45775324/hregion-45775324.1731422630187, exclude list is [], retry=0 2024-11-12T14:43:50,204 DEBUG [AsyncFSWAL-26-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37395,DS-7317e4d9-cef6-4b85-a947-106c3c917da9,DISK] 2024-11-12T14:43:50,205 DEBUG [AsyncFSWAL-26-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45975,DS-80b047b6-34d2-41c0-99c3-5cd61ca40075,DISK] 2024-11-12T14:43:50,211 DEBUG [AsyncFSWAL-26-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36175,DS-31e7ba0a-e37c-4fe6-8d2a-a4846cefa408,DISK] 2024-11-12T14:43:50,254 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-45775324/hregion-45775324.1731422630187 2024-11-12T14:43:50,255 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40227:40227),(127.0.0.1/127.0.0.1:43643:43643),(127.0.0.1/127.0.0.1:36027:36027)] 2024-11-12T14:43:50,255 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 8f5ccc576c0beb19653167612ebd769c, NAME => 'testCompactedBulkLoadedFiles,,1731422630181.8f5ccc576c0beb19653167612ebd769c.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testCompactedBulkLoadedFiles', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40559/hbase 2024-11-12T14:43:50,267 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37395 is added to blk_1073741916_1094 (size=63) 2024-11-12T14:43:50,268 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36175 is added to blk_1073741916_1094 (size=63) 2024-11-12T14:43:50,269 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45975 is added to blk_1073741916_1094 (size=63) 2024-11-12T14:43:50,271 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testCompactedBulkLoadedFiles,,1731422630181.8f5ccc576c0beb19653167612ebd769c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T14:43:50,273 INFO [StoreOpener-8f5ccc576c0beb19653167612ebd769c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 8f5ccc576c0beb19653167612ebd769c 2024-11-12T14:43:50,275 INFO [StoreOpener-8f5ccc576c0beb19653167612ebd769c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8f5ccc576c0beb19653167612ebd769c columnFamilyName a 2024-11-12T14:43:50,275 DEBUG [StoreOpener-8f5ccc576c0beb19653167612ebd769c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:43:50,276 INFO [StoreOpener-8f5ccc576c0beb19653167612ebd769c-1 {}] regionserver.HStore(327): Store=8f5ccc576c0beb19653167612ebd769c/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T14:43:50,276 INFO [StoreOpener-8f5ccc576c0beb19653167612ebd769c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 8f5ccc576c0beb19653167612ebd769c 2024-11-12T14:43:50,278 INFO [StoreOpener-8f5ccc576c0beb19653167612ebd769c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8f5ccc576c0beb19653167612ebd769c columnFamilyName b 2024-11-12T14:43:50,278 DEBUG [StoreOpener-8f5ccc576c0beb19653167612ebd769c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:43:50,280 INFO [StoreOpener-8f5ccc576c0beb19653167612ebd769c-1 {}] regionserver.HStore(327): Store=8f5ccc576c0beb19653167612ebd769c/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T14:43:50,280 INFO [StoreOpener-8f5ccc576c0beb19653167612ebd769c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 8f5ccc576c0beb19653167612ebd769c 2024-11-12T14:43:50,282 INFO [StoreOpener-8f5ccc576c0beb19653167612ebd769c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8f5ccc576c0beb19653167612ebd769c columnFamilyName c 2024-11-12T14:43:50,282 DEBUG [StoreOpener-8f5ccc576c0beb19653167612ebd769c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:43:50,283 INFO [StoreOpener-8f5ccc576c0beb19653167612ebd769c-1 {}] regionserver.HStore(327): Store=8f5ccc576c0beb19653167612ebd769c/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T14:43:50,283 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 8f5ccc576c0beb19653167612ebd769c 2024-11-12T14:43:50,284 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40559/hbase/data/default/testCompactedBulkLoadedFiles/8f5ccc576c0beb19653167612ebd769c 2024-11-12T14:43:50,285 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40559/hbase/data/default/testCompactedBulkLoadedFiles/8f5ccc576c0beb19653167612ebd769c 2024-11-12T14:43:50,286 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 8f5ccc576c0beb19653167612ebd769c 2024-11-12T14:43:50,286 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 8f5ccc576c0beb19653167612ebd769c 2024-11-12T14:43:50,287 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testCompactedBulkLoadedFiles descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-12T14:43:50,288 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 8f5ccc576c0beb19653167612ebd769c 2024-11-12T14:43:50,291 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40559/hbase/data/default/testCompactedBulkLoadedFiles/8f5ccc576c0beb19653167612ebd769c/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-12T14:43:50,292 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 8f5ccc576c0beb19653167612ebd769c; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67041456, jitterRate=-0.0010044574737548828}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-12T14:43:50,293 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 8f5ccc576c0beb19653167612ebd769c: Writing region info on filesystem at 1731422630272Initializing all the Stores at 1731422630273 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731422630273Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731422630273Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731422630273Cleaning up temporary data from old regions at 1731422630286 (+13 ms)Region opened successfully at 1731422630293 (+7 ms) 2024-11-12T14:43:50,293 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 8f5ccc576c0beb19653167612ebd769c, disabling compactions & flushes 2024-11-12T14:43:50,293 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testCompactedBulkLoadedFiles,,1731422630181.8f5ccc576c0beb19653167612ebd769c. 2024-11-12T14:43:50,293 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testCompactedBulkLoadedFiles,,1731422630181.8f5ccc576c0beb19653167612ebd769c. 2024-11-12T14:43:50,293 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testCompactedBulkLoadedFiles,,1731422630181.8f5ccc576c0beb19653167612ebd769c. after waiting 0 ms 2024-11-12T14:43:50,293 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testCompactedBulkLoadedFiles,,1731422630181.8f5ccc576c0beb19653167612ebd769c. 2024-11-12T14:43:50,294 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testCompactedBulkLoadedFiles,,1731422630181.8f5ccc576c0beb19653167612ebd769c. 2024-11-12T14:43:50,294 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 8f5ccc576c0beb19653167612ebd769c: Waiting for close lock at 1731422630293Disabling compacts and flushes for region at 1731422630293Disabling writes for close at 1731422630293Writing region close event to WAL at 1731422630294 (+1 ms)Closed at 1731422630294 2024-11-12T14:43:50,300 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37395 is added to blk_1073741915_1093 (size=175) 2024-11-12T14:43:50,300 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36175 is added to blk_1073741915_1093 (size=175) 2024-11-12T14:43:50,304 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45975 is added to blk_1073741915_1093 (size=175) 2024-11-12T14:43:50,310 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-12T14:43:50,310 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-45775324:(num 1731422630187) 2024-11-12T14:43:50,310 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-12T14:43:50,313 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:40559/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731422630179, archiveDir=hdfs://localhost:40559/hbase/oldWALs, maxLogs=32 2024-11-12T14:43:50,331 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731422630179/wal.1731422630314, exclude list is [], retry=0 2024-11-12T14:43:50,335 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45975,DS-80b047b6-34d2-41c0-99c3-5cd61ca40075,DISK] 2024-11-12T14:43:50,335 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36175,DS-31e7ba0a-e37c-4fe6-8d2a-a4846cefa408,DISK] 2024-11-12T14:43:50,335 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37395,DS-7317e4d9-cef6-4b85-a947-106c3c917da9,DISK] 2024-11-12T14:43:50,381 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731422630179/wal.1731422630314 2024-11-12T14:43:50,381 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43643:43643),(127.0.0.1/127.0.0.1:36027:36027),(127.0.0.1/127.0.0.1:40227:40227)] 2024-11-12T14:43:50,382 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 8f5ccc576c0beb19653167612ebd769c, NAME => 'testCompactedBulkLoadedFiles,,1731422630181.8f5ccc576c0beb19653167612ebd769c.', STARTKEY => '', ENDKEY => ''} 2024-11-12T14:43:50,382 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testCompactedBulkLoadedFiles,,1731422630181.8f5ccc576c0beb19653167612ebd769c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T14:43:50,382 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 8f5ccc576c0beb19653167612ebd769c 2024-11-12T14:43:50,382 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 8f5ccc576c0beb19653167612ebd769c 2024-11-12T14:43:50,384 INFO [StoreOpener-8f5ccc576c0beb19653167612ebd769c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 8f5ccc576c0beb19653167612ebd769c 2024-11-12T14:43:50,384 INFO [StoreOpener-8f5ccc576c0beb19653167612ebd769c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8f5ccc576c0beb19653167612ebd769c columnFamilyName a 2024-11-12T14:43:50,384 DEBUG [StoreOpener-8f5ccc576c0beb19653167612ebd769c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:43:50,385 INFO [StoreOpener-8f5ccc576c0beb19653167612ebd769c-1 {}] regionserver.HStore(327): Store=8f5ccc576c0beb19653167612ebd769c/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T14:43:50,385 INFO [StoreOpener-8f5ccc576c0beb19653167612ebd769c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 8f5ccc576c0beb19653167612ebd769c 2024-11-12T14:43:50,386 INFO [StoreOpener-8f5ccc576c0beb19653167612ebd769c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8f5ccc576c0beb19653167612ebd769c columnFamilyName b 2024-11-12T14:43:50,386 DEBUG [StoreOpener-8f5ccc576c0beb19653167612ebd769c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:43:50,386 INFO [StoreOpener-8f5ccc576c0beb19653167612ebd769c-1 {}] regionserver.HStore(327): Store=8f5ccc576c0beb19653167612ebd769c/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T14:43:50,387 INFO [StoreOpener-8f5ccc576c0beb19653167612ebd769c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 8f5ccc576c0beb19653167612ebd769c 2024-11-12T14:43:50,387 INFO [StoreOpener-8f5ccc576c0beb19653167612ebd769c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8f5ccc576c0beb19653167612ebd769c columnFamilyName c 2024-11-12T14:43:50,388 DEBUG [StoreOpener-8f5ccc576c0beb19653167612ebd769c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:43:50,388 INFO [StoreOpener-8f5ccc576c0beb19653167612ebd769c-1 {}] regionserver.HStore(327): Store=8f5ccc576c0beb19653167612ebd769c/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T14:43:50,388 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 8f5ccc576c0beb19653167612ebd769c 2024-11-12T14:43:50,389 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40559/hbase/data/default/testCompactedBulkLoadedFiles/8f5ccc576c0beb19653167612ebd769c 2024-11-12T14:43:50,390 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40559/hbase/data/default/testCompactedBulkLoadedFiles/8f5ccc576c0beb19653167612ebd769c 2024-11-12T14:43:50,391 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 8f5ccc576c0beb19653167612ebd769c 2024-11-12T14:43:50,391 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 8f5ccc576c0beb19653167612ebd769c 2024-11-12T14:43:50,391 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testCompactedBulkLoadedFiles descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-12T14:43:50,393 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 8f5ccc576c0beb19653167612ebd769c 2024-11-12T14:43:50,395 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 8f5ccc576c0beb19653167612ebd769c; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63957364, jitterRate=-0.04696100950241089}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-12T14:43:50,396 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 8f5ccc576c0beb19653167612ebd769c: Writing region info on filesystem at 1731422630382Initializing all the Stores at 1731422630383 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731422630383Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731422630383Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731422630383Cleaning up temporary data from old regions at 1731422630391 (+8 ms)Region opened successfully at 1731422630395 (+4 ms) 2024-11-12T14:43:50,401 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40559/hbase/testCompactedBulkLoadedFiles/hfile0 is 32, key is 000/a:a/1731422630400/Put/seqid=0 2024-11-12T14:43:50,415 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37395 is added to blk_1073741918_1096 (size=4875) 2024-11-12T14:43:50,416 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45975 is added to blk_1073741918_1096 (size=4875) 2024-11-12T14:43:50,417 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36175 is added to blk_1073741918_1096 (size=4875) 2024-11-12T14:43:50,421 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40559/hbase/testCompactedBulkLoadedFiles/hfile1 is 32, key is 100/a:a/1731422630420/Put/seqid=0 2024-11-12T14:43:50,446 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45975 is added to blk_1073741919_1097 (size=4875) 2024-11-12T14:43:50,447 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36175 is added to blk_1073741919_1097 (size=4875) 2024-11-12T14:43:50,447 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37395 is added to blk_1073741919_1097 (size=4875) 2024-11-12T14:43:50,453 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40559/hbase/testCompactedBulkLoadedFiles/hfile2 is 32, key is 200/a:a/1731422630451/Put/seqid=0 2024-11-12T14:43:50,471 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37395 is added to blk_1073741920_1098 (size=4875) 2024-11-12T14:43:50,471 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45975 is added to blk_1073741920_1098 (size=4875) 2024-11-12T14:43:50,472 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36175 is added to blk_1073741920_1098 (size=4875) 2024-11-12T14:43:50,472 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:40559/hbase/testCompactedBulkLoadedFiles/hfile0 for inclusion in 8f5ccc576c0beb19653167612ebd769c/a 2024-11-12T14:43:50,478 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first=000 last=050 2024-11-12T14:43:50,478 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-11-12T14:43:50,478 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:40559/hbase/testCompactedBulkLoadedFiles/hfile1 for inclusion in 8f5ccc576c0beb19653167612ebd769c/a 2024-11-12T14:43:50,481 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first=100 last=150 2024-11-12T14:43:50,481 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-11-12T14:43:50,482 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:40559/hbase/testCompactedBulkLoadedFiles/hfile2 for inclusion in 8f5ccc576c0beb19653167612ebd769c/a 2024-11-12T14:43:50,485 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first=200 last=250 2024-11-12T14:43:50,485 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-11-12T14:43:50,485 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 8f5ccc576c0beb19653167612ebd769c 3/3 column families, dataSize=51 B heapSize=896 B 2024-11-12T14:43:50,501 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40559/hbase/data/default/testCompactedBulkLoadedFiles/8f5ccc576c0beb19653167612ebd769c/.tmp/a/7e0cbcf78c1b4e389bd4fa545af9755c is 55, key is testCompactedBulkLoadedFiles/a:a/1731422630396/Put/seqid=0 2024-11-12T14:43:50,517 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45975 is added to blk_1073741921_1099 (size=5107) 2024-11-12T14:43:50,518 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37395 is added to blk_1073741921_1099 (size=5107) 2024-11-12T14:43:50,519 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36175 is added to blk_1073741921_1099 (size=5107) 2024-11-12T14:43:50,521 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51 B at sequenceid=4 (bloomFilter=true), to=hdfs://localhost:40559/hbase/data/default/testCompactedBulkLoadedFiles/8f5ccc576c0beb19653167612ebd769c/.tmp/a/7e0cbcf78c1b4e389bd4fa545af9755c 2024-11-12T14:43:50,529 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40559/hbase/data/default/testCompactedBulkLoadedFiles/8f5ccc576c0beb19653167612ebd769c/.tmp/a/7e0cbcf78c1b4e389bd4fa545af9755c as hdfs://localhost:40559/hbase/data/default/testCompactedBulkLoadedFiles/8f5ccc576c0beb19653167612ebd769c/a/7e0cbcf78c1b4e389bd4fa545af9755c 2024-11-12T14:43:50,538 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40559/hbase/data/default/testCompactedBulkLoadedFiles/8f5ccc576c0beb19653167612ebd769c/a/7e0cbcf78c1b4e389bd4fa545af9755c, entries=1, sequenceid=4, filesize=5.0 K 2024-11-12T14:43:50,539 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~51 B/51, heapSize ~368 B/368, currentSize=0 B/0 for 8f5ccc576c0beb19653167612ebd769c in 54ms, sequenceid=4, compaction requested=false 2024-11-12T14:43:50,539 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 8f5ccc576c0beb19653167612ebd769c: 2024-11-12T14:43:50,542 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40559/hbase/testCompactedBulkLoadedFiles/hfile0 as hdfs://localhost:40559/hbase/data/default/testCompactedBulkLoadedFiles/8f5ccc576c0beb19653167612ebd769c/a/277979a10c7e4b5db5b47eb6045b185a_SeqId_4_ 2024-11-12T14:43:50,543 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40559/hbase/testCompactedBulkLoadedFiles/hfile1 as hdfs://localhost:40559/hbase/data/default/testCompactedBulkLoadedFiles/8f5ccc576c0beb19653167612ebd769c/a/4bf983fca58d4385a1f5cd866e2d269b_SeqId_4_ 2024-11-12T14:43:50,544 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40559/hbase/testCompactedBulkLoadedFiles/hfile2 as hdfs://localhost:40559/hbase/data/default/testCompactedBulkLoadedFiles/8f5ccc576c0beb19653167612ebd769c/a/83911d73947f435d806d180a48eff312_SeqId_4_ 2024-11-12T14:43:50,545 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:40559/hbase/testCompactedBulkLoadedFiles/hfile0 into 8f5ccc576c0beb19653167612ebd769c/a as hdfs://localhost:40559/hbase/data/default/testCompactedBulkLoadedFiles/8f5ccc576c0beb19653167612ebd769c/a/277979a10c7e4b5db5b47eb6045b185a_SeqId_4_ - updating store file list. 2024-11-12T14:43:50,552 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for 277979a10c7e4b5db5b47eb6045b185a_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-12T14:43:50,552 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:40559/hbase/data/default/testCompactedBulkLoadedFiles/8f5ccc576c0beb19653167612ebd769c/a/277979a10c7e4b5db5b47eb6045b185a_SeqId_4_ into 8f5ccc576c0beb19653167612ebd769c/a 2024-11-12T14:43:50,552 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:40559/hbase/testCompactedBulkLoadedFiles/hfile0 into 8f5ccc576c0beb19653167612ebd769c/a (new location: hdfs://localhost:40559/hbase/data/default/testCompactedBulkLoadedFiles/8f5ccc576c0beb19653167612ebd769c/a/277979a10c7e4b5db5b47eb6045b185a_SeqId_4_) 2024-11-12T14:43:50,553 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:40559/hbase/testCompactedBulkLoadedFiles/hfile1 into 8f5ccc576c0beb19653167612ebd769c/a as hdfs://localhost:40559/hbase/data/default/testCompactedBulkLoadedFiles/8f5ccc576c0beb19653167612ebd769c/a/4bf983fca58d4385a1f5cd866e2d269b_SeqId_4_ - updating store file list. 2024-11-12T14:43:50,560 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for 4bf983fca58d4385a1f5cd866e2d269b_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-12T14:43:50,560 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:40559/hbase/data/default/testCompactedBulkLoadedFiles/8f5ccc576c0beb19653167612ebd769c/a/4bf983fca58d4385a1f5cd866e2d269b_SeqId_4_ into 8f5ccc576c0beb19653167612ebd769c/a 2024-11-12T14:43:50,560 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:40559/hbase/testCompactedBulkLoadedFiles/hfile1 into 8f5ccc576c0beb19653167612ebd769c/a (new location: hdfs://localhost:40559/hbase/data/default/testCompactedBulkLoadedFiles/8f5ccc576c0beb19653167612ebd769c/a/4bf983fca58d4385a1f5cd866e2d269b_SeqId_4_) 2024-11-12T14:43:50,561 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:40559/hbase/testCompactedBulkLoadedFiles/hfile2 into 8f5ccc576c0beb19653167612ebd769c/a as hdfs://localhost:40559/hbase/data/default/testCompactedBulkLoadedFiles/8f5ccc576c0beb19653167612ebd769c/a/83911d73947f435d806d180a48eff312_SeqId_4_ - updating store file list. 2024-11-12T14:43:50,565 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for 83911d73947f435d806d180a48eff312_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-12T14:43:50,565 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:40559/hbase/data/default/testCompactedBulkLoadedFiles/8f5ccc576c0beb19653167612ebd769c/a/83911d73947f435d806d180a48eff312_SeqId_4_ into 8f5ccc576c0beb19653167612ebd769c/a 2024-11-12T14:43:50,566 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:40559/hbase/testCompactedBulkLoadedFiles/hfile2 into 8f5ccc576c0beb19653167612ebd769c/a (new location: hdfs://localhost:40559/hbase/data/default/testCompactedBulkLoadedFiles/8f5ccc576c0beb19653167612ebd769c/a/83911d73947f435d806d180a48eff312_SeqId_4_) 2024-11-12T14:43:50,589 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-12T14:43:50,589 DEBUG [Time-limited test {}] regionserver.HStore(1541): 8f5ccc576c0beb19653167612ebd769c/a is initiating major compaction (all files) 2024-11-12T14:43:50,589 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of 8f5ccc576c0beb19653167612ebd769c/a in testCompactedBulkLoadedFiles,,1731422630181.8f5ccc576c0beb19653167612ebd769c. 2024-11-12T14:43:50,590 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40559/hbase/data/default/testCompactedBulkLoadedFiles/8f5ccc576c0beb19653167612ebd769c/a/7e0cbcf78c1b4e389bd4fa545af9755c, hdfs://localhost:40559/hbase/data/default/testCompactedBulkLoadedFiles/8f5ccc576c0beb19653167612ebd769c/a/277979a10c7e4b5db5b47eb6045b185a_SeqId_4_, hdfs://localhost:40559/hbase/data/default/testCompactedBulkLoadedFiles/8f5ccc576c0beb19653167612ebd769c/a/4bf983fca58d4385a1f5cd866e2d269b_SeqId_4_, hdfs://localhost:40559/hbase/data/default/testCompactedBulkLoadedFiles/8f5ccc576c0beb19653167612ebd769c/a/83911d73947f435d806d180a48eff312_SeqId_4_] into tmpdir=hdfs://localhost:40559/hbase/data/default/testCompactedBulkLoadedFiles/8f5ccc576c0beb19653167612ebd769c/.tmp, totalSize=19.3 K 2024-11-12T14:43:50,590 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 7e0cbcf78c1b4e389bd4fa545af9755c, keycount=1, bloomtype=ROW, size=5.0 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=1731422630396 2024-11-12T14:43:50,591 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 277979a10c7e4b5db5b47eb6045b185a_SeqId_4_, keycount=10, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=-9223372036854775808 2024-11-12T14:43:50,591 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 4bf983fca58d4385a1f5cd866e2d269b_SeqId_4_, keycount=10, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=-9223372036854775808 2024-11-12T14:43:50,592 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 83911d73947f435d806d180a48eff312_SeqId_4_, keycount=10, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=-9223372036854775808 2024-11-12T14:43:50,612 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40559/hbase/data/default/testCompactedBulkLoadedFiles/8f5ccc576c0beb19653167612ebd769c/.tmp/a/a95a0b4d9ec946088a6b468c31fc144a is 55, key is testCompactedBulkLoadedFiles/a:a/1731422630396/Put/seqid=0 2024-11-12T14:43:50,641 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45975 is added to blk_1073741922_1100 (size=6154) 2024-11-12T14:43:50,643 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36175 is added to blk_1073741922_1100 (size=6154) 2024-11-12T14:43:50,644 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37395 is added to blk_1073741922_1100 (size=6154) 2024-11-12T14:43:50,656 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40559/hbase/data/default/testCompactedBulkLoadedFiles/8f5ccc576c0beb19653167612ebd769c/.tmp/a/a95a0b4d9ec946088a6b468c31fc144a as hdfs://localhost:40559/hbase/data/default/testCompactedBulkLoadedFiles/8f5ccc576c0beb19653167612ebd769c/a/a95a0b4d9ec946088a6b468c31fc144a 2024-11-12T14:43:50,667 INFO [Time-limited test {}] regionserver.HStore(1337): Completed major compaction of 4 (all) file(s) in 8f5ccc576c0beb19653167612ebd769c/a of 8f5ccc576c0beb19653167612ebd769c into a95a0b4d9ec946088a6b468c31fc144a(size=6.0 K), total size for store is 6.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T14:43:50,667 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for 8f5ccc576c0beb19653167612ebd769c: 2024-11-12T14:43:50,667 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 0 store files, 0 compacting, 0 eligible, 16 blocking 2024-11-12T14:43:50,667 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 0 store files, 0 compacting, 0 eligible, 16 blocking 2024-11-12T14:43:50,719 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:40559/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731422630179/wal.1731422630314, size=0 (0bytes) 2024-11-12T14:43:50,719 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:40559/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731422630179/wal.1731422630314 might be still open, length is 0 2024-11-12T14:43:50,719 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:40559/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731422630179/wal.1731422630314 2024-11-12T14:43:50,721 WARN [IPC Server handler 0 on default port 40559 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731422630179/wal.1731422630314 has not been closed. Lease recovery is in progress. RecoveryId = 1101 for block blk_1073741917_1095 2024-11-12T14:43:50,722 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:40559/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731422630179/wal.1731422630314 after 3ms 2024-11-12T14:43:51,024 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-425049348_22 at /127.0.0.1:41392 [Receiving block BP-1980882744-172.17.0.2-1731422596004:blk_1073741917_1095] {}] datanode.DataXceiver(331): 127.0.0.1:45975:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:41392 dst: /127.0.0.1:45975 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:45975 remote=/127.0.0.1:41392]. Total timeout mills is 60000, 59643 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:43:51,024 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-425049348_22 at /127.0.0.1:60168 [Receiving block BP-1980882744-172.17.0.2-1731422596004:blk_1073741917_1095] {}] datanode.DataXceiver(331): 127.0.0.1:37395:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:60168 dst: /127.0.0.1:37395 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:43:51,025 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-425049348_22 at /127.0.0.1:59794 [Receiving block BP-1980882744-172.17.0.2-1731422596004:blk_1073741917_1095] {}] datanode.DataXceiver(331): 127.0.0.1:36175:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:59794 dst: /127.0.0.1:36175 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:43:51,028 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45975 is added to blk_1073741917_1101 (size=1794) 2024-11-12T14:43:51,028 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36175 is added to blk_1073741917_1101 (size=1794) 2024-11-12T14:43:54,132 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-12T14:43:54,722 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:40559/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731422630179/wal.1731422630314 after 4003ms 2024-11-12T14:43:54,756 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-12T14:43:54,756 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:40559/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731422630179/wal.1731422630314 took 4037ms 2024-11-12T14:43:54,758 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:40559/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731422630179/wal.1731422630314; continuing. 2024-11-12T14:43:54,758 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:40559/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731422630179/wal.1731422630314 so closing down 2024-11-12T14:43:54,758 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-12T14:43:54,759 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1731422630314.temp 2024-11-12T14:43:54,787 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:40559/hbase/data/default/testCompactedBulkLoadedFiles/8f5ccc576c0beb19653167612ebd769c/recovered.edits/0000000000000000003-wal.1731422630314.temp 2024-11-12T14:43:54,787 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-12T14:43:54,794 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36175 is added to blk_1073741923_1102 (size=800) 2024-11-12T14:43:54,795 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37395 is added to blk_1073741923_1102 (size=800) 2024-11-12T14:43:54,795 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45975 is added to blk_1073741923_1102 (size=800) 2024-11-12T14:43:54,796 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:40559/hbase/data/default/testCompactedBulkLoadedFiles/8f5ccc576c0beb19653167612ebd769c/recovered.edits/0000000000000000003-wal.1731422630314.temp (wrote 2 edits, skipped 0 edits in 0 ms) 2024-11-12T14:43:54,798 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:40559/hbase/data/default/testCompactedBulkLoadedFiles/8f5ccc576c0beb19653167612ebd769c/recovered.edits/0000000000000000003-wal.1731422630314.temp to hdfs://localhost:40559/hbase/data/default/testCompactedBulkLoadedFiles/8f5ccc576c0beb19653167612ebd769c/recovered.edits/0000000000000000008 2024-11-12T14:43:54,798 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 5 edits across 1 Regions in 42 ms; skipped=3; WAL=hdfs://localhost:40559/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731422630179/wal.1731422630314, size=0, length=0, corrupted=false, cancelled=false 2024-11-12T14:43:54,798 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:40559/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731422630179/wal.1731422630314, journal: Splitting hdfs://localhost:40559/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731422630179/wal.1731422630314, size=0 (0bytes) at 1731422630719Finishing writing output for hdfs://localhost:40559/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731422630179/wal.1731422630314 so closing down at 1731422634758 (+4039 ms)Creating recovered edits writer path=hdfs://localhost:40559/hbase/data/default/testCompactedBulkLoadedFiles/8f5ccc576c0beb19653167612ebd769c/recovered.edits/0000000000000000003-wal.1731422630314.temp at 1731422634787 (+29 ms)3 split writer threads finished at 1731422634787Closed recovered edits writer path=hdfs://localhost:40559/hbase/data/default/testCompactedBulkLoadedFiles/8f5ccc576c0beb19653167612ebd769c/recovered.edits/0000000000000000003-wal.1731422630314.temp (wrote 2 edits, skipped 0 edits in 0 ms) at 1731422634796 (+9 ms)Rename recovered edits hdfs://localhost:40559/hbase/data/default/testCompactedBulkLoadedFiles/8f5ccc576c0beb19653167612ebd769c/recovered.edits/0000000000000000003-wal.1731422630314.temp to hdfs://localhost:40559/hbase/data/default/testCompactedBulkLoadedFiles/8f5ccc576c0beb19653167612ebd769c/recovered.edits/0000000000000000008 at 1731422634798 (+2 ms)Processed 5 edits across 1 Regions in 42 ms; skipped=3; WAL=hdfs://localhost:40559/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731422630179/wal.1731422630314, size=0, length=0, corrupted=false, cancelled=false at 1731422634798 2024-11-12T14:43:54,800 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:40559/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731422630179/wal.1731422630314 to hdfs://localhost:40559/hbase/oldWALs/wal.1731422630314 2024-11-12T14:43:54,801 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:40559/hbase/data/default/testCompactedBulkLoadedFiles/8f5ccc576c0beb19653167612ebd769c/recovered.edits/0000000000000000008 2024-11-12T14:43:54,801 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-12T14:43:54,802 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:40559/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731422630179, archiveDir=hdfs://localhost:40559/hbase/oldWALs, maxLogs=32 2024-11-12T14:43:54,814 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731422630179/wal.1731422634803, exclude list is [], retry=0 2024-11-12T14:43:54,818 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36175,DS-31e7ba0a-e37c-4fe6-8d2a-a4846cefa408,DISK] 2024-11-12T14:43:54,818 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37395,DS-7317e4d9-cef6-4b85-a947-106c3c917da9,DISK] 2024-11-12T14:43:54,818 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45975,DS-80b047b6-34d2-41c0-99c3-5cd61ca40075,DISK] 2024-11-12T14:43:54,847 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731422630179/wal.1731422634803 2024-11-12T14:43:54,851 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36027:36027),(127.0.0.1/127.0.0.1:40227:40227),(127.0.0.1/127.0.0.1:43643:43643)] 2024-11-12T14:43:54,852 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 8f5ccc576c0beb19653167612ebd769c, NAME => 'testCompactedBulkLoadedFiles,,1731422630181.8f5ccc576c0beb19653167612ebd769c.', STARTKEY => '', ENDKEY => ''} 2024-11-12T14:43:54,852 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testCompactedBulkLoadedFiles,,1731422630181.8f5ccc576c0beb19653167612ebd769c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T14:43:54,852 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 8f5ccc576c0beb19653167612ebd769c 2024-11-12T14:43:54,852 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 8f5ccc576c0beb19653167612ebd769c 2024-11-12T14:43:54,853 INFO [StoreOpener-8f5ccc576c0beb19653167612ebd769c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 8f5ccc576c0beb19653167612ebd769c 2024-11-12T14:43:54,854 INFO [StoreOpener-8f5ccc576c0beb19653167612ebd769c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8f5ccc576c0beb19653167612ebd769c columnFamilyName a 2024-11-12T14:43:54,854 DEBUG [StoreOpener-8f5ccc576c0beb19653167612ebd769c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:43:54,864 DEBUG [StoreFileOpener-8f5ccc576c0beb19653167612ebd769c-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for 277979a10c7e4b5db5b47eb6045b185a_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-12T14:43:54,864 DEBUG [StoreOpener-8f5ccc576c0beb19653167612ebd769c-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:40559/hbase/data/default/testCompactedBulkLoadedFiles/8f5ccc576c0beb19653167612ebd769c/a/277979a10c7e4b5db5b47eb6045b185a_SeqId_4_ 2024-11-12T14:43:54,868 DEBUG [StoreFileOpener-8f5ccc576c0beb19653167612ebd769c-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for 4bf983fca58d4385a1f5cd866e2d269b_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-12T14:43:54,869 DEBUG [StoreOpener-8f5ccc576c0beb19653167612ebd769c-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:40559/hbase/data/default/testCompactedBulkLoadedFiles/8f5ccc576c0beb19653167612ebd769c/a/4bf983fca58d4385a1f5cd866e2d269b_SeqId_4_ 2024-11-12T14:43:54,873 DEBUG [StoreOpener-8f5ccc576c0beb19653167612ebd769c-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:40559/hbase/data/default/testCompactedBulkLoadedFiles/8f5ccc576c0beb19653167612ebd769c/a/7e0cbcf78c1b4e389bd4fa545af9755c 2024-11-12T14:43:54,876 DEBUG [StoreFileOpener-8f5ccc576c0beb19653167612ebd769c-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for 83911d73947f435d806d180a48eff312_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-12T14:43:54,876 DEBUG [StoreOpener-8f5ccc576c0beb19653167612ebd769c-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:40559/hbase/data/default/testCompactedBulkLoadedFiles/8f5ccc576c0beb19653167612ebd769c/a/83911d73947f435d806d180a48eff312_SeqId_4_ 2024-11-12T14:43:54,880 DEBUG [StoreOpener-8f5ccc576c0beb19653167612ebd769c-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:40559/hbase/data/default/testCompactedBulkLoadedFiles/8f5ccc576c0beb19653167612ebd769c/a/a95a0b4d9ec946088a6b468c31fc144a 2024-11-12T14:43:54,880 WARN [StoreOpener-8f5ccc576c0beb19653167612ebd769c-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:40559/hbase/data/default/testCompactedBulkLoadedFiles/8f5ccc576c0beb19653167612ebd769c/a/277979a10c7e4b5db5b47eb6045b185a_SeqId_4_ from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@3198c58d 2024-11-12T14:43:54,880 WARN [StoreOpener-8f5ccc576c0beb19653167612ebd769c-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:40559/hbase/data/default/testCompactedBulkLoadedFiles/8f5ccc576c0beb19653167612ebd769c/a/4bf983fca58d4385a1f5cd866e2d269b_SeqId_4_ from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@3198c58d 2024-11-12T14:43:54,880 WARN [StoreOpener-8f5ccc576c0beb19653167612ebd769c-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:40559/hbase/data/default/testCompactedBulkLoadedFiles/8f5ccc576c0beb19653167612ebd769c/a/7e0cbcf78c1b4e389bd4fa545af9755c from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@3198c58d 2024-11-12T14:43:54,880 WARN [StoreOpener-8f5ccc576c0beb19653167612ebd769c-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:40559/hbase/data/default/testCompactedBulkLoadedFiles/8f5ccc576c0beb19653167612ebd769c/a/83911d73947f435d806d180a48eff312_SeqId_4_ from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@3198c58d 2024-11-12T14:43:54,880 DEBUG [StoreOpener-8f5ccc576c0beb19653167612ebd769c-1 {}] regionserver.StoreEngine(327): Moving the files [hdfs://localhost:40559/hbase/data/default/testCompactedBulkLoadedFiles/8f5ccc576c0beb19653167612ebd769c/a/277979a10c7e4b5db5b47eb6045b185a_SeqId_4_, hdfs://localhost:40559/hbase/data/default/testCompactedBulkLoadedFiles/8f5ccc576c0beb19653167612ebd769c/a/4bf983fca58d4385a1f5cd866e2d269b_SeqId_4_, hdfs://localhost:40559/hbase/data/default/testCompactedBulkLoadedFiles/8f5ccc576c0beb19653167612ebd769c/a/7e0cbcf78c1b4e389bd4fa545af9755c, hdfs://localhost:40559/hbase/data/default/testCompactedBulkLoadedFiles/8f5ccc576c0beb19653167612ebd769c/a/83911d73947f435d806d180a48eff312_SeqId_4_] to archive 2024-11-12T14:43:54,882 DEBUG [StoreOpener-8f5ccc576c0beb19653167612ebd769c-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-12T14:43:54,883 DEBUG [StoreOpener-8f5ccc576c0beb19653167612ebd769c-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40559/hbase/data/default/testCompactedBulkLoadedFiles/8f5ccc576c0beb19653167612ebd769c/a/277979a10c7e4b5db5b47eb6045b185a_SeqId_4_ to hdfs://localhost:40559/hbase/archive/data/default/testCompactedBulkLoadedFiles/8f5ccc576c0beb19653167612ebd769c/a/277979a10c7e4b5db5b47eb6045b185a_SeqId_4_ 2024-11-12T14:43:54,885 DEBUG [StoreOpener-8f5ccc576c0beb19653167612ebd769c-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40559/hbase/data/default/testCompactedBulkLoadedFiles/8f5ccc576c0beb19653167612ebd769c/a/4bf983fca58d4385a1f5cd866e2d269b_SeqId_4_ to hdfs://localhost:40559/hbase/archive/data/default/testCompactedBulkLoadedFiles/8f5ccc576c0beb19653167612ebd769c/a/4bf983fca58d4385a1f5cd866e2d269b_SeqId_4_ 2024-11-12T14:43:54,886 DEBUG [StoreOpener-8f5ccc576c0beb19653167612ebd769c-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40559/hbase/data/default/testCompactedBulkLoadedFiles/8f5ccc576c0beb19653167612ebd769c/a/7e0cbcf78c1b4e389bd4fa545af9755c to hdfs://localhost:40559/hbase/archive/data/default/testCompactedBulkLoadedFiles/8f5ccc576c0beb19653167612ebd769c/a/7e0cbcf78c1b4e389bd4fa545af9755c 2024-11-12T14:43:54,887 DEBUG [StoreOpener-8f5ccc576c0beb19653167612ebd769c-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40559/hbase/data/default/testCompactedBulkLoadedFiles/8f5ccc576c0beb19653167612ebd769c/a/83911d73947f435d806d180a48eff312_SeqId_4_ to hdfs://localhost:40559/hbase/archive/data/default/testCompactedBulkLoadedFiles/8f5ccc576c0beb19653167612ebd769c/a/83911d73947f435d806d180a48eff312_SeqId_4_ 2024-11-12T14:43:54,888 INFO [StoreOpener-8f5ccc576c0beb19653167612ebd769c-1 {}] regionserver.HStore(327): Store=8f5ccc576c0beb19653167612ebd769c/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T14:43:54,888 INFO [StoreOpener-8f5ccc576c0beb19653167612ebd769c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 8f5ccc576c0beb19653167612ebd769c 2024-11-12T14:43:54,889 INFO [StoreOpener-8f5ccc576c0beb19653167612ebd769c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8f5ccc576c0beb19653167612ebd769c columnFamilyName b 2024-11-12T14:43:54,889 DEBUG [StoreOpener-8f5ccc576c0beb19653167612ebd769c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:43:54,890 INFO [StoreOpener-8f5ccc576c0beb19653167612ebd769c-1 {}] regionserver.HStore(327): Store=8f5ccc576c0beb19653167612ebd769c/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T14:43:54,890 INFO [StoreOpener-8f5ccc576c0beb19653167612ebd769c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 8f5ccc576c0beb19653167612ebd769c 2024-11-12T14:43:54,891 INFO [StoreOpener-8f5ccc576c0beb19653167612ebd769c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8f5ccc576c0beb19653167612ebd769c columnFamilyName c 2024-11-12T14:43:54,892 DEBUG [StoreOpener-8f5ccc576c0beb19653167612ebd769c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:43:54,892 INFO [StoreOpener-8f5ccc576c0beb19653167612ebd769c-1 {}] regionserver.HStore(327): Store=8f5ccc576c0beb19653167612ebd769c/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T14:43:54,892 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 8f5ccc576c0beb19653167612ebd769c 2024-11-12T14:43:54,893 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40559/hbase/data/default/testCompactedBulkLoadedFiles/8f5ccc576c0beb19653167612ebd769c 2024-11-12T14:43:54,894 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:40559/hbase/data/default/testCompactedBulkLoadedFiles/8f5ccc576c0beb19653167612ebd769c 2024-11-12T14:43:54,895 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:40559/hbase/data/default/testCompactedBulkLoadedFiles/8f5ccc576c0beb19653167612ebd769c/recovered.edits/0000000000000000008 2024-11-12T14:43:54,929 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-12T14:43:54,932 DEBUG [Time-limited test {}] regionserver.HRegion(5836): 8f5ccc576c0beb19653167612ebd769c : Replaying compaction marker table_name: "testCompactedBulkLoadedFiles" encoded_region_name: "8f5ccc576c0beb19653167612ebd769c" family_name: "a" compaction_input: "7e0cbcf78c1b4e389bd4fa545af9755c" compaction_input: "277979a10c7e4b5db5b47eb6045b185a_SeqId_4_" compaction_input: "4bf983fca58d4385a1f5cd866e2d269b_SeqId_4_" compaction_input: "83911d73947f435d806d180a48eff312_SeqId_4_" compaction_output: "a95a0b4d9ec946088a6b468c31fc144a" store_home_dir: "a" region_name: "testCompactedBulkLoadedFiles,,1731422630181.8f5ccc576c0beb19653167612ebd769c." with seqId=9223372036854775807 and lastReplayedOpenRegionSeqId=-1 2024-11-12T14:43:54,932 DEBUG [Time-limited test {}] regionserver.HStore(1354): Completing compaction from the WAL marker 2024-11-12T14:43:54,932 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 0, skipped 2, firstSequenceIdInLog=3, maxSequenceIdInLog=8, path=hdfs://localhost:40559/hbase/data/default/testCompactedBulkLoadedFiles/8f5ccc576c0beb19653167612ebd769c/recovered.edits/0000000000000000008 2024-11-12T14:43:54,933 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:40559/hbase/data/default/testCompactedBulkLoadedFiles/8f5ccc576c0beb19653167612ebd769c/recovered.edits/0000000000000000008 2024-11-12T14:43:54,934 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 8f5ccc576c0beb19653167612ebd769c 2024-11-12T14:43:54,934 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 8f5ccc576c0beb19653167612ebd769c 2024-11-12T14:43:54,935 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testCompactedBulkLoadedFiles descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-12T14:43:54,936 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 8f5ccc576c0beb19653167612ebd769c 2024-11-12T14:43:54,939 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40559/hbase/data/default/testCompactedBulkLoadedFiles/8f5ccc576c0beb19653167612ebd769c/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-12T14:43:54,942 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 8f5ccc576c0beb19653167612ebd769c; next sequenceid=9; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64111378, jitterRate=-0.044666022062301636}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-12T14:43:54,942 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 8f5ccc576c0beb19653167612ebd769c: Writing region info on filesystem at 1731422634852Initializing all the Stores at 1731422634853 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731422634853Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731422634853Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731422634853Cleaning up temporary data from old regions at 1731422634934 (+81 ms)Region opened successfully at 1731422634942 (+8 ms) 2024-11-12T14:43:54,944 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 8f5ccc576c0beb19653167612ebd769c, disabling compactions & flushes 2024-11-12T14:43:54,944 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testCompactedBulkLoadedFiles,,1731422630181.8f5ccc576c0beb19653167612ebd769c. 2024-11-12T14:43:54,944 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testCompactedBulkLoadedFiles,,1731422630181.8f5ccc576c0beb19653167612ebd769c. 2024-11-12T14:43:54,944 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testCompactedBulkLoadedFiles,,1731422630181.8f5ccc576c0beb19653167612ebd769c. after waiting 0 ms 2024-11-12T14:43:54,944 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testCompactedBulkLoadedFiles,,1731422630181.8f5ccc576c0beb19653167612ebd769c. 2024-11-12T14:43:54,948 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testCompactedBulkLoadedFiles,,1731422630181.8f5ccc576c0beb19653167612ebd769c. 2024-11-12T14:43:54,948 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 8f5ccc576c0beb19653167612ebd769c: Waiting for close lock at 1731422634944Disabling compacts and flushes for region at 1731422634944Disabling writes for close at 1731422634944Writing region close event to WAL at 1731422634947 (+3 ms)Closed at 1731422634947 2024-11-12T14:43:54,950 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36175 is added to blk_1073741924_1103 (size=175) 2024-11-12T14:43:54,950 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45975 is added to blk_1073741924_1103 (size=175) 2024-11-12T14:43:54,950 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37395 is added to blk_1073741924_1103 (size=175) 2024-11-12T14:43:54,952 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-12T14:43:54,952 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1731422634803) 2024-11-12T14:43:54,972 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testCompactedBulkLoadedFiles Thread=444 (was 431) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1222692828_22 at /127.0.0.1:60242 [Waiting for operation #5] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-26-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1685805399) connection to localhost/127.0.0.1:40559 from jenkinstestCompactedBulkLoadedFiles java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: LeaseRenewer:jenkinstestCompactedBulkLoadedFiles@localhost:40559 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-26-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-26-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1222692828_22 at /127.0.0.1:41470 [Waiting for operation #11] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1222692828_22 at /127.0.0.1:59890 [Waiting for operation #3] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1348 (was 1264) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=335 (was 251) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=7754 (was 8186) 2024-11-12T14:43:54,972 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1348 is superior to 1024 2024-11-12T14:43:54,988 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsWrittenViaHRegion Thread=444, OpenFileDescriptor=1348, MaxFileDescriptor=1048576, SystemLoadAverage=335, ProcessCount=11, AvailableMemoryMB=7752 2024-11-12T14:43:54,988 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1348 is superior to 1024 2024-11-12T14:43:55,010 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-12T14:43:55,012 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-12T14:43:55,013 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-12T14:43:55,015 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-86937406, suffix=, logDir=hdfs://localhost:40559/hbase/WALs/hregion-86937406, archiveDir=hdfs://localhost:40559/hbase/oldWALs, maxLogs=32 2024-11-12T14:43:55,031 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-86937406/hregion-86937406.1731422635016, exclude list is [], retry=0 2024-11-12T14:43:55,034 DEBUG [AsyncFSWAL-28-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37395,DS-7317e4d9-cef6-4b85-a947-106c3c917da9,DISK] 2024-11-12T14:43:55,035 DEBUG [AsyncFSWAL-28-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36175,DS-31e7ba0a-e37c-4fe6-8d2a-a4846cefa408,DISK] 2024-11-12T14:43:55,035 DEBUG [AsyncFSWAL-28-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45975,DS-80b047b6-34d2-41c0-99c3-5cd61ca40075,DISK] 2024-11-12T14:43:55,068 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-86937406/hregion-86937406.1731422635016 2024-11-12T14:43:55,068 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40227:40227),(127.0.0.1/127.0.0.1:36027:36027),(127.0.0.1/127.0.0.1:43643:43643)] 2024-11-12T14:43:55,068 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 88bbcedda8de78f6ce3bcc3cde9eac6f, NAME => 'testReplayEditsWrittenViaHRegion,,1731422635010.88bbcedda8de78f6ce3bcc3cde9eac6f.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenViaHRegion', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40559/hbase 2024-11-12T14:43:55,083 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36175 is added to blk_1073741926_1105 (size=67) 2024-11-12T14:43:55,083 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45975 is added to blk_1073741926_1105 (size=67) 2024-11-12T14:43:55,083 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37395 is added to blk_1073741926_1105 (size=67) 2024-11-12T14:43:55,084 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1731422635010.88bbcedda8de78f6ce3bcc3cde9eac6f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T14:43:55,094 INFO [StoreOpener-88bbcedda8de78f6ce3bcc3cde9eac6f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 88bbcedda8de78f6ce3bcc3cde9eac6f 2024-11-12T14:43:55,096 INFO [StoreOpener-88bbcedda8de78f6ce3bcc3cde9eac6f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 88bbcedda8de78f6ce3bcc3cde9eac6f columnFamilyName a 2024-11-12T14:43:55,096 DEBUG [StoreOpener-88bbcedda8de78f6ce3bcc3cde9eac6f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:43:55,096 INFO [StoreOpener-88bbcedda8de78f6ce3bcc3cde9eac6f-1 {}] regionserver.HStore(327): Store=88bbcedda8de78f6ce3bcc3cde9eac6f/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T14:43:55,097 INFO [StoreOpener-88bbcedda8de78f6ce3bcc3cde9eac6f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 88bbcedda8de78f6ce3bcc3cde9eac6f 2024-11-12T14:43:55,098 INFO [StoreOpener-88bbcedda8de78f6ce3bcc3cde9eac6f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 88bbcedda8de78f6ce3bcc3cde9eac6f columnFamilyName b 2024-11-12T14:43:55,098 DEBUG [StoreOpener-88bbcedda8de78f6ce3bcc3cde9eac6f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:43:55,099 INFO [StoreOpener-88bbcedda8de78f6ce3bcc3cde9eac6f-1 {}] regionserver.HStore(327): Store=88bbcedda8de78f6ce3bcc3cde9eac6f/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T14:43:55,099 INFO [StoreOpener-88bbcedda8de78f6ce3bcc3cde9eac6f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 88bbcedda8de78f6ce3bcc3cde9eac6f 2024-11-12T14:43:55,100 INFO [StoreOpener-88bbcedda8de78f6ce3bcc3cde9eac6f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 88bbcedda8de78f6ce3bcc3cde9eac6f columnFamilyName c 2024-11-12T14:43:55,100 DEBUG [StoreOpener-88bbcedda8de78f6ce3bcc3cde9eac6f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:43:55,101 INFO [StoreOpener-88bbcedda8de78f6ce3bcc3cde9eac6f-1 {}] regionserver.HStore(327): Store=88bbcedda8de78f6ce3bcc3cde9eac6f/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T14:43:55,101 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 88bbcedda8de78f6ce3bcc3cde9eac6f 2024-11-12T14:43:55,102 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40559/hbase/data/default/testReplayEditsWrittenViaHRegion/88bbcedda8de78f6ce3bcc3cde9eac6f 2024-11-12T14:43:55,102 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40559/hbase/data/default/testReplayEditsWrittenViaHRegion/88bbcedda8de78f6ce3bcc3cde9eac6f 2024-11-12T14:43:55,103 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 88bbcedda8de78f6ce3bcc3cde9eac6f 2024-11-12T14:43:55,103 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 88bbcedda8de78f6ce3bcc3cde9eac6f 2024-11-12T14:43:55,104 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-12T14:43:55,105 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 88bbcedda8de78f6ce3bcc3cde9eac6f 2024-11-12T14:43:55,107 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40559/hbase/data/default/testReplayEditsWrittenViaHRegion/88bbcedda8de78f6ce3bcc3cde9eac6f/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-12T14:43:55,107 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 88bbcedda8de78f6ce3bcc3cde9eac6f; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68601244, jitterRate=0.022238194942474365}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-12T14:43:55,108 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 88bbcedda8de78f6ce3bcc3cde9eac6f: Writing region info on filesystem at 1731422635084Initializing all the Stores at 1731422635085 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731422635085Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731422635094 (+9 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731422635094Cleaning up temporary data from old regions at 1731422635103 (+9 ms)Region opened successfully at 1731422635107 (+4 ms) 2024-11-12T14:43:55,108 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 88bbcedda8de78f6ce3bcc3cde9eac6f, disabling compactions & flushes 2024-11-12T14:43:55,108 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1731422635010.88bbcedda8de78f6ce3bcc3cde9eac6f. 2024-11-12T14:43:55,108 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1731422635010.88bbcedda8de78f6ce3bcc3cde9eac6f. 2024-11-12T14:43:55,108 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1731422635010.88bbcedda8de78f6ce3bcc3cde9eac6f. after waiting 0 ms 2024-11-12T14:43:55,108 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1731422635010.88bbcedda8de78f6ce3bcc3cde9eac6f. 2024-11-12T14:43:55,109 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1731422635010.88bbcedda8de78f6ce3bcc3cde9eac6f. 2024-11-12T14:43:55,110 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 88bbcedda8de78f6ce3bcc3cde9eac6f: Waiting for close lock at 1731422635108Disabling compacts and flushes for region at 1731422635108Disabling writes for close at 1731422635108Writing region close event to WAL at 1731422635109 (+1 ms)Closed at 1731422635109 2024-11-12T14:43:55,115 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45975 is added to blk_1073741925_1104 (size=175) 2024-11-12T14:43:55,115 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36175 is added to blk_1073741925_1104 (size=175) 2024-11-12T14:43:55,116 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37395 is added to blk_1073741925_1104 (size=175) 2024-11-12T14:43:55,119 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-12T14:43:55,119 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-86937406:(num 1731422635016) 2024-11-12T14:43:55,119 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-12T14:43:55,121 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:40559/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731422635009, archiveDir=hdfs://localhost:40559/hbase/oldWALs, maxLogs=32 2024-11-12T14:43:55,141 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731422635009/wal.1731422635122, exclude list is [], retry=0 2024-11-12T14:43:55,145 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45975,DS-80b047b6-34d2-41c0-99c3-5cd61ca40075,DISK] 2024-11-12T14:43:55,145 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37395,DS-7317e4d9-cef6-4b85-a947-106c3c917da9,DISK] 2024-11-12T14:43:55,146 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36175,DS-31e7ba0a-e37c-4fe6-8d2a-a4846cefa408,DISK] 2024-11-12T14:43:55,183 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731422635009/wal.1731422635122 2024-11-12T14:43:55,190 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43643:43643),(127.0.0.1/127.0.0.1:40227:40227),(127.0.0.1/127.0.0.1:36027:36027)] 2024-11-12T14:43:55,190 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 88bbcedda8de78f6ce3bcc3cde9eac6f, NAME => 'testReplayEditsWrittenViaHRegion,,1731422635010.88bbcedda8de78f6ce3bcc3cde9eac6f.', STARTKEY => '', ENDKEY => ''} 2024-11-12T14:43:55,190 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1731422635010.88bbcedda8de78f6ce3bcc3cde9eac6f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T14:43:55,190 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 88bbcedda8de78f6ce3bcc3cde9eac6f 2024-11-12T14:43:55,190 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 88bbcedda8de78f6ce3bcc3cde9eac6f 2024-11-12T14:43:55,191 INFO [StoreOpener-88bbcedda8de78f6ce3bcc3cde9eac6f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 88bbcedda8de78f6ce3bcc3cde9eac6f 2024-11-12T14:43:55,192 INFO [StoreOpener-88bbcedda8de78f6ce3bcc3cde9eac6f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 88bbcedda8de78f6ce3bcc3cde9eac6f columnFamilyName a 2024-11-12T14:43:55,192 DEBUG [StoreOpener-88bbcedda8de78f6ce3bcc3cde9eac6f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:43:55,193 INFO [StoreOpener-88bbcedda8de78f6ce3bcc3cde9eac6f-1 {}] regionserver.HStore(327): Store=88bbcedda8de78f6ce3bcc3cde9eac6f/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T14:43:55,193 INFO [StoreOpener-88bbcedda8de78f6ce3bcc3cde9eac6f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 88bbcedda8de78f6ce3bcc3cde9eac6f 2024-11-12T14:43:55,194 INFO [StoreOpener-88bbcedda8de78f6ce3bcc3cde9eac6f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 88bbcedda8de78f6ce3bcc3cde9eac6f columnFamilyName b 2024-11-12T14:43:55,194 DEBUG [StoreOpener-88bbcedda8de78f6ce3bcc3cde9eac6f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:43:55,194 INFO [StoreOpener-88bbcedda8de78f6ce3bcc3cde9eac6f-1 {}] regionserver.HStore(327): Store=88bbcedda8de78f6ce3bcc3cde9eac6f/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T14:43:55,195 INFO [StoreOpener-88bbcedda8de78f6ce3bcc3cde9eac6f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 88bbcedda8de78f6ce3bcc3cde9eac6f 2024-11-12T14:43:55,196 INFO [StoreOpener-88bbcedda8de78f6ce3bcc3cde9eac6f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 88bbcedda8de78f6ce3bcc3cde9eac6f columnFamilyName c 2024-11-12T14:43:55,196 DEBUG [StoreOpener-88bbcedda8de78f6ce3bcc3cde9eac6f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:43:55,196 INFO [StoreOpener-88bbcedda8de78f6ce3bcc3cde9eac6f-1 {}] regionserver.HStore(327): Store=88bbcedda8de78f6ce3bcc3cde9eac6f/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T14:43:55,196 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 88bbcedda8de78f6ce3bcc3cde9eac6f 2024-11-12T14:43:55,197 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40559/hbase/data/default/testReplayEditsWrittenViaHRegion/88bbcedda8de78f6ce3bcc3cde9eac6f 2024-11-12T14:43:55,199 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40559/hbase/data/default/testReplayEditsWrittenViaHRegion/88bbcedda8de78f6ce3bcc3cde9eac6f 2024-11-12T14:43:55,200 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 88bbcedda8de78f6ce3bcc3cde9eac6f 2024-11-12T14:43:55,200 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 88bbcedda8de78f6ce3bcc3cde9eac6f 2024-11-12T14:43:55,200 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-12T14:43:55,202 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 88bbcedda8de78f6ce3bcc3cde9eac6f 2024-11-12T14:43:55,203 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 88bbcedda8de78f6ce3bcc3cde9eac6f; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60861101, jitterRate=-0.09309892356395721}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-12T14:43:55,203 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 88bbcedda8de78f6ce3bcc3cde9eac6f: Writing region info on filesystem at 1731422635190Initializing all the Stores at 1731422635191 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731422635191Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731422635191Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731422635191Cleaning up temporary data from old regions at 1731422635200 (+9 ms)Region opened successfully at 1731422635203 (+3 ms) 2024-11-12T14:43:55,223 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 88bbcedda8de78f6ce3bcc3cde9eac6f 3/3 column families, dataSize=870 B heapSize=2.31 KB 2024-11-12T14:43:55,246 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40559/hbase/data/default/testReplayEditsWrittenViaHRegion/88bbcedda8de78f6ce3bcc3cde9eac6f/.tmp/a/1ef72724db554a86b4e868a99ce7f723 is 91, key is testReplayEditsWrittenViaHRegion/a:x0/1731422635203/Put/seqid=0 2024-11-12T14:43:55,260 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36175 is added to blk_1073741928_1107 (size=5958) 2024-11-12T14:43:55,260 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37395 is added to blk_1073741928_1107 (size=5958) 2024-11-12T14:43:55,262 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45975 is added to blk_1073741928_1107 (size=5958) 2024-11-12T14:43:55,263 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:40559/hbase/data/default/testReplayEditsWrittenViaHRegion/88bbcedda8de78f6ce3bcc3cde9eac6f/.tmp/a/1ef72724db554a86b4e868a99ce7f723 2024-11-12T14:43:55,269 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40559/hbase/data/default/testReplayEditsWrittenViaHRegion/88bbcedda8de78f6ce3bcc3cde9eac6f/.tmp/a/1ef72724db554a86b4e868a99ce7f723 as hdfs://localhost:40559/hbase/data/default/testReplayEditsWrittenViaHRegion/88bbcedda8de78f6ce3bcc3cde9eac6f/a/1ef72724db554a86b4e868a99ce7f723 2024-11-12T14:43:55,274 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40559/hbase/data/default/testReplayEditsWrittenViaHRegion/88bbcedda8de78f6ce3bcc3cde9eac6f/a/1ef72724db554a86b4e868a99ce7f723, entries=10, sequenceid=13, filesize=5.8 K 2024-11-12T14:43:55,275 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~870 B/870, heapSize ~1.80 KB/1840, currentSize=0 B/0 for 88bbcedda8de78f6ce3bcc3cde9eac6f in 51ms, sequenceid=13, compaction requested=false 2024-11-12T14:43:55,275 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 88bbcedda8de78f6ce3bcc3cde9eac6f: 2024-11-12T14:43:55,294 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 88bbcedda8de78f6ce3bcc3cde9eac6f, disabling compactions & flushes 2024-11-12T14:43:55,294 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1731422635010.88bbcedda8de78f6ce3bcc3cde9eac6f. 2024-11-12T14:43:55,294 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1731422635010.88bbcedda8de78f6ce3bcc3cde9eac6f. 2024-11-12T14:43:55,294 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1731422635010.88bbcedda8de78f6ce3bcc3cde9eac6f. after waiting 0 ms 2024-11-12T14:43:55,294 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1731422635010.88bbcedda8de78f6ce3bcc3cde9eac6f. 2024-11-12T14:43:55,295 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 1740 in region testReplayEditsWrittenViaHRegion,,1731422635010.88bbcedda8de78f6ce3bcc3cde9eac6f. 2024-11-12T14:43:55,295 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1731422635010.88bbcedda8de78f6ce3bcc3cde9eac6f. 2024-11-12T14:43:55,295 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 88bbcedda8de78f6ce3bcc3cde9eac6f: Waiting for close lock at 1731422635294Disabling compacts and flushes for region at 1731422635294Disabling writes for close at 1731422635294Writing region close event to WAL at 1731422635295 (+1 ms)Closed at 1731422635295 2024-11-12T14:43:55,298 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37395 is added to blk_1073741927_1106 (size=6243) 2024-11-12T14:43:55,298 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731422635009/wal.1731422635122 not finished, retry = 0 2024-11-12T14:43:55,298 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45975 is added to blk_1073741927_1106 (size=6243) 2024-11-12T14:43:55,298 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36175 is added to blk_1073741927_1106 (size=6243) 2024-11-12T14:43:55,420 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:40559/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731422635009/wal.1731422635122, size=6.1 K (6243bytes) 2024-11-12T14:43:55,420 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:40559/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731422635009/wal.1731422635122 2024-11-12T14:43:55,421 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:40559/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731422635009/wal.1731422635122 after 0ms 2024-11-12T14:43:55,451 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-12T14:43:55,452 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:40559/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731422635009/wal.1731422635122 took 32ms 2024-11-12T14:43:55,454 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:40559/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731422635009/wal.1731422635122 so closing down 2024-11-12T14:43:55,454 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-12T14:43:55,456 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1731422635122.temp 2024-11-12T14:43:55,485 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:40559/hbase/data/default/testReplayEditsWrittenViaHRegion/88bbcedda8de78f6ce3bcc3cde9eac6f/recovered.edits/0000000000000000003-wal.1731422635122.temp 2024-11-12T14:43:55,485 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-12T14:43:55,495 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45975 is added to blk_1073741929_1108 (size=5575) 2024-11-12T14:43:55,497 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37395 is added to blk_1073741929_1108 (size=5575) 2024-11-12T14:43:55,497 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36175 is added to blk_1073741929_1108 (size=5575) 2024-11-12T14:43:55,498 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:40559/hbase/data/default/testReplayEditsWrittenViaHRegion/88bbcedda8de78f6ce3bcc3cde9eac6f/recovered.edits/0000000000000000003-wal.1731422635122.temp (wrote 30 edits, skipped 0 edits in 0 ms) 2024-11-12T14:43:55,500 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:40559/hbase/data/default/testReplayEditsWrittenViaHRegion/88bbcedda8de78f6ce3bcc3cde9eac6f/recovered.edits/0000000000000000003-wal.1731422635122.temp to hdfs://localhost:40559/hbase/data/default/testReplayEditsWrittenViaHRegion/88bbcedda8de78f6ce3bcc3cde9eac6f/recovered.edits/0000000000000000035 2024-11-12T14:43:55,500 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 32 edits across 1 Regions in 48 ms; skipped=2; WAL=hdfs://localhost:40559/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731422635009/wal.1731422635122, size=6.1 K, length=6243, corrupted=false, cancelled=false 2024-11-12T14:43:55,500 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:40559/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731422635009/wal.1731422635122, journal: Splitting hdfs://localhost:40559/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731422635009/wal.1731422635122, size=6.1 K (6243bytes) at 1731422635420Finishing writing output for hdfs://localhost:40559/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731422635009/wal.1731422635122 so closing down at 1731422635454 (+34 ms)Creating recovered edits writer path=hdfs://localhost:40559/hbase/data/default/testReplayEditsWrittenViaHRegion/88bbcedda8de78f6ce3bcc3cde9eac6f/recovered.edits/0000000000000000003-wal.1731422635122.temp at 1731422635485 (+31 ms)3 split writer threads finished at 1731422635485Closed recovered edits writer path=hdfs://localhost:40559/hbase/data/default/testReplayEditsWrittenViaHRegion/88bbcedda8de78f6ce3bcc3cde9eac6f/recovered.edits/0000000000000000003-wal.1731422635122.temp (wrote 30 edits, skipped 0 edits in 0 ms) at 1731422635498 (+13 ms)Rename recovered edits hdfs://localhost:40559/hbase/data/default/testReplayEditsWrittenViaHRegion/88bbcedda8de78f6ce3bcc3cde9eac6f/recovered.edits/0000000000000000003-wal.1731422635122.temp to hdfs://localhost:40559/hbase/data/default/testReplayEditsWrittenViaHRegion/88bbcedda8de78f6ce3bcc3cde9eac6f/recovered.edits/0000000000000000035 at 1731422635500 (+2 ms)Processed 32 edits across 1 Regions in 48 ms; skipped=2; WAL=hdfs://localhost:40559/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731422635009/wal.1731422635122, size=6.1 K, length=6243, corrupted=false, cancelled=false at 1731422635500 2024-11-12T14:43:55,502 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:40559/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731422635009/wal.1731422635122 to hdfs://localhost:40559/hbase/oldWALs/wal.1731422635122 2024-11-12T14:43:55,503 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:40559/hbase/data/default/testReplayEditsWrittenViaHRegion/88bbcedda8de78f6ce3bcc3cde9eac6f/recovered.edits/0000000000000000035 2024-11-12T14:43:55,503 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-12T14:43:55,505 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:40559/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731422635009, archiveDir=hdfs://localhost:40559/hbase/oldWALs, maxLogs=32 2024-11-12T14:43:55,529 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731422635009/wal.1731422635506, exclude list is [], retry=0 2024-11-12T14:43:55,532 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36175,DS-31e7ba0a-e37c-4fe6-8d2a-a4846cefa408,DISK] 2024-11-12T14:43:55,532 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45975,DS-80b047b6-34d2-41c0-99c3-5cd61ca40075,DISK] 2024-11-12T14:43:55,533 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37395,DS-7317e4d9-cef6-4b85-a947-106c3c917da9,DISK] 2024-11-12T14:43:55,584 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731422635009/wal.1731422635506 2024-11-12T14:43:55,585 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36027:36027),(127.0.0.1/127.0.0.1:43643:43643),(127.0.0.1/127.0.0.1:40227:40227)] 2024-11-12T14:43:55,585 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 88bbcedda8de78f6ce3bcc3cde9eac6f, NAME => 'testReplayEditsWrittenViaHRegion,,1731422635010.88bbcedda8de78f6ce3bcc3cde9eac6f.', STARTKEY => '', ENDKEY => ''} 2024-11-12T14:43:55,585 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1731422635010.88bbcedda8de78f6ce3bcc3cde9eac6f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T14:43:55,585 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 88bbcedda8de78f6ce3bcc3cde9eac6f 2024-11-12T14:43:55,586 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 88bbcedda8de78f6ce3bcc3cde9eac6f 2024-11-12T14:43:55,587 INFO [StoreOpener-88bbcedda8de78f6ce3bcc3cde9eac6f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 88bbcedda8de78f6ce3bcc3cde9eac6f 2024-11-12T14:43:55,589 INFO [StoreOpener-88bbcedda8de78f6ce3bcc3cde9eac6f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 88bbcedda8de78f6ce3bcc3cde9eac6f columnFamilyName a 2024-11-12T14:43:55,589 DEBUG [StoreOpener-88bbcedda8de78f6ce3bcc3cde9eac6f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:43:55,596 DEBUG [StoreOpener-88bbcedda8de78f6ce3bcc3cde9eac6f-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:40559/hbase/data/default/testReplayEditsWrittenViaHRegion/88bbcedda8de78f6ce3bcc3cde9eac6f/a/1ef72724db554a86b4e868a99ce7f723 2024-11-12T14:43:55,596 INFO [StoreOpener-88bbcedda8de78f6ce3bcc3cde9eac6f-1 {}] regionserver.HStore(327): Store=88bbcedda8de78f6ce3bcc3cde9eac6f/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T14:43:55,596 INFO [StoreOpener-88bbcedda8de78f6ce3bcc3cde9eac6f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 88bbcedda8de78f6ce3bcc3cde9eac6f 2024-11-12T14:43:55,598 INFO [StoreOpener-88bbcedda8de78f6ce3bcc3cde9eac6f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 88bbcedda8de78f6ce3bcc3cde9eac6f columnFamilyName b 2024-11-12T14:43:55,598 DEBUG [StoreOpener-88bbcedda8de78f6ce3bcc3cde9eac6f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:43:55,600 INFO [StoreOpener-88bbcedda8de78f6ce3bcc3cde9eac6f-1 {}] regionserver.HStore(327): Store=88bbcedda8de78f6ce3bcc3cde9eac6f/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T14:43:55,600 INFO [StoreOpener-88bbcedda8de78f6ce3bcc3cde9eac6f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 88bbcedda8de78f6ce3bcc3cde9eac6f 2024-11-12T14:43:55,605 INFO [StoreOpener-88bbcedda8de78f6ce3bcc3cde9eac6f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 88bbcedda8de78f6ce3bcc3cde9eac6f columnFamilyName c 2024-11-12T14:43:55,605 DEBUG [StoreOpener-88bbcedda8de78f6ce3bcc3cde9eac6f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:43:55,609 INFO [StoreOpener-88bbcedda8de78f6ce3bcc3cde9eac6f-1 {}] regionserver.HStore(327): Store=88bbcedda8de78f6ce3bcc3cde9eac6f/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T14:43:55,609 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 88bbcedda8de78f6ce3bcc3cde9eac6f 2024-11-12T14:43:55,610 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40559/hbase/data/default/testReplayEditsWrittenViaHRegion/88bbcedda8de78f6ce3bcc3cde9eac6f 2024-11-12T14:43:55,612 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:40559/hbase/data/default/testReplayEditsWrittenViaHRegion/88bbcedda8de78f6ce3bcc3cde9eac6f 2024-11-12T14:43:55,612 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:40559/hbase/data/default/testReplayEditsWrittenViaHRegion/88bbcedda8de78f6ce3bcc3cde9eac6f/recovered.edits/0000000000000000035 2024-11-12T14:43:55,653 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-12T14:43:55,654 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 20, skipped 10, firstSequenceIdInLog=3, maxSequenceIdInLog=35, path=hdfs://localhost:40559/hbase/data/default/testReplayEditsWrittenViaHRegion/88bbcedda8de78f6ce3bcc3cde9eac6f/recovered.edits/0000000000000000035 2024-11-12T14:43:55,655 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 88bbcedda8de78f6ce3bcc3cde9eac6f 3/3 column families, dataSize=1.70 KB heapSize=3.88 KB 2024-11-12T14:43:55,676 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40559/hbase/data/default/testReplayEditsWrittenViaHRegion/88bbcedda8de78f6ce3bcc3cde9eac6f/.tmp/b/c6d6ebb188904ad5b71917e09d882ef2 is 91, key is testReplayEditsWrittenViaHRegion/b:x0/1731422635275/Put/seqid=0 2024-11-12T14:43:55,685 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45975 is added to blk_1073741931_1110 (size=5958) 2024-11-12T14:43:55,686 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36175 is added to blk_1073741931_1110 (size=5958) 2024-11-12T14:43:55,686 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37395 is added to blk_1073741931_1110 (size=5958) 2024-11-12T14:43:55,690 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=35 (bloomFilter=true), to=hdfs://localhost:40559/hbase/data/default/testReplayEditsWrittenViaHRegion/88bbcedda8de78f6ce3bcc3cde9eac6f/.tmp/b/c6d6ebb188904ad5b71917e09d882ef2 2024-11-12T14:43:55,715 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40559/hbase/data/default/testReplayEditsWrittenViaHRegion/88bbcedda8de78f6ce3bcc3cde9eac6f/.tmp/c/9f3f73ddb0644a8a93a0ca2c39528ef6 is 91, key is testReplayEditsWrittenViaHRegion/c:x0/1731422635282/Put/seqid=0 2024-11-12T14:43:55,732 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37395 is added to blk_1073741932_1111 (size=5958) 2024-11-12T14:43:55,733 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36175 is added to blk_1073741932_1111 (size=5958) 2024-11-12T14:43:55,733 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45975 is added to blk_1073741932_1111 (size=5958) 2024-11-12T14:43:55,734 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=35 (bloomFilter=true), to=hdfs://localhost:40559/hbase/data/default/testReplayEditsWrittenViaHRegion/88bbcedda8de78f6ce3bcc3cde9eac6f/.tmp/c/9f3f73ddb0644a8a93a0ca2c39528ef6 2024-11-12T14:43:55,748 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40559/hbase/data/default/testReplayEditsWrittenViaHRegion/88bbcedda8de78f6ce3bcc3cde9eac6f/.tmp/b/c6d6ebb188904ad5b71917e09d882ef2 as hdfs://localhost:40559/hbase/data/default/testReplayEditsWrittenViaHRegion/88bbcedda8de78f6ce3bcc3cde9eac6f/b/c6d6ebb188904ad5b71917e09d882ef2 2024-11-12T14:43:55,753 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40559/hbase/data/default/testReplayEditsWrittenViaHRegion/88bbcedda8de78f6ce3bcc3cde9eac6f/b/c6d6ebb188904ad5b71917e09d882ef2, entries=10, sequenceid=35, filesize=5.8 K 2024-11-12T14:43:55,754 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40559/hbase/data/default/testReplayEditsWrittenViaHRegion/88bbcedda8de78f6ce3bcc3cde9eac6f/.tmp/c/9f3f73ddb0644a8a93a0ca2c39528ef6 as hdfs://localhost:40559/hbase/data/default/testReplayEditsWrittenViaHRegion/88bbcedda8de78f6ce3bcc3cde9eac6f/c/9f3f73ddb0644a8a93a0ca2c39528ef6 2024-11-12T14:43:55,759 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40559/hbase/data/default/testReplayEditsWrittenViaHRegion/88bbcedda8de78f6ce3bcc3cde9eac6f/c/9f3f73ddb0644a8a93a0ca2c39528ef6, entries=10, sequenceid=35, filesize=5.8 K 2024-11-12T14:43:55,759 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~1.70 KB/1740, heapSize ~3.59 KB/3680, currentSize=0 B/0 for 88bbcedda8de78f6ce3bcc3cde9eac6f in 105ms, sequenceid=35, compaction requested=false; wal=null 2024-11-12T14:43:55,760 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:40559/hbase/data/default/testReplayEditsWrittenViaHRegion/88bbcedda8de78f6ce3bcc3cde9eac6f/recovered.edits/0000000000000000035 2024-11-12T14:43:55,761 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 88bbcedda8de78f6ce3bcc3cde9eac6f 2024-11-12T14:43:55,761 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 88bbcedda8de78f6ce3bcc3cde9eac6f 2024-11-12T14:43:55,761 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-12T14:43:55,763 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 88bbcedda8de78f6ce3bcc3cde9eac6f 2024-11-12T14:43:55,765 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40559/hbase/data/default/testReplayEditsWrittenViaHRegion/88bbcedda8de78f6ce3bcc3cde9eac6f/recovered.edits/35.seqid, newMaxSeqId=35, maxSeqId=1 2024-11-12T14:43:55,766 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 88bbcedda8de78f6ce3bcc3cde9eac6f; next sequenceid=36; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62727388, jitterRate=-0.06528908014297485}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-12T14:43:55,766 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 88bbcedda8de78f6ce3bcc3cde9eac6f: Writing region info on filesystem at 1731422635586Initializing all the Stores at 1731422635587 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731422635587Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731422635587Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731422635587Obtaining lock to block concurrent updates at 1731422635655 (+68 ms)Preparing flush snapshotting stores in 88bbcedda8de78f6ce3bcc3cde9eac6f at 1731422635655Finished memstore snapshotting testReplayEditsWrittenViaHRegion,,1731422635010.88bbcedda8de78f6ce3bcc3cde9eac6f., syncing WAL and waiting on mvcc, flushsize=dataSize=1740, getHeapSize=3920, getOffHeapSize=0, getCellsCount=20 at 1731422635655Flushing stores of testReplayEditsWrittenViaHRegion,,1731422635010.88bbcedda8de78f6ce3bcc3cde9eac6f. at 1731422635655Flushing 88bbcedda8de78f6ce3bcc3cde9eac6f/b: creating writer at 1731422635655Flushing 88bbcedda8de78f6ce3bcc3cde9eac6f/b: appending metadata at 1731422635675 (+20 ms)Flushing 88bbcedda8de78f6ce3bcc3cde9eac6f/b: closing flushed file at 1731422635675Flushing 88bbcedda8de78f6ce3bcc3cde9eac6f/c: creating writer at 1731422635699 (+24 ms)Flushing 88bbcedda8de78f6ce3bcc3cde9eac6f/c: appending metadata at 1731422635714 (+15 ms)Flushing 88bbcedda8de78f6ce3bcc3cde9eac6f/c: closing flushed file at 1731422635714Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6f650eaf: reopening flushed file at 1731422635746 (+32 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2a1d66e9: reopening flushed file at 1731422635753 (+7 ms)Finished flush of dataSize ~1.70 KB/1740, heapSize ~3.59 KB/3680, currentSize=0 B/0 for 88bbcedda8de78f6ce3bcc3cde9eac6f in 105ms, sequenceid=35, compaction requested=false; wal=null at 1731422635759 (+6 ms)Cleaning up temporary data from old regions at 1731422635761 (+2 ms)Region opened successfully at 1731422635766 (+5 ms) 2024-11-12T14:43:55,859 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:40559/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731422635009/wal.1731422635506, size=0 (0bytes) 2024-11-12T14:43:55,859 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:40559/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731422635009/wal.1731422635506 might be still open, length is 0 2024-11-12T14:43:55,859 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:40559/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731422635009/wal.1731422635506 2024-11-12T14:43:55,859 WARN [IPC Server handler 4 on default port 40559 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731422635009/wal.1731422635506 has not been closed. Lease recovery is in progress. RecoveryId = 1112 for block blk_1073741930_1109 2024-11-12T14:43:55,859 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:40559/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731422635009/wal.1731422635506 after 0ms 2024-11-12T14:43:57,023 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-425049348_22 at /127.0.0.1:59966 [Receiving block BP-1980882744-172.17.0.2-1731422596004:blk_1073741930_1109] {}] datanode.DataXceiver(331): 127.0.0.1:36175:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:59966 dst: /127.0.0.1:36175 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:36175 remote=/127.0.0.1:59966]. Total timeout mills is 60000, 58784 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:43:57,024 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-425049348_22 at /127.0.0.1:41530 [Receiving block BP-1980882744-172.17.0.2-1731422596004:blk_1073741930_1109] {}] datanode.DataXceiver(331): 127.0.0.1:45975:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:41530 dst: /127.0.0.1:45975 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:43:57,024 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-425049348_22 at /127.0.0.1:60292 [Receiving block BP-1980882744-172.17.0.2-1731422596004:blk_1073741930_1109] {}] datanode.DataXceiver(331): 127.0.0.1:37395:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:60292 dst: /127.0.0.1:37395 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:43:57,027 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36175 is added to blk_1073741930_1112 (size=5567) 2024-11-12T14:43:57,028 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45975 is added to blk_1073741930_1112 (size=5567) 2024-11-12T14:43:59,365 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testDatalossWhenInputError 2024-11-12T14:43:59,365 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testDatalossWhenInputError Metrics about Tables on a single HBase RegionServer 2024-11-12T14:43:59,366 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testCompactedBulkLoadedFiles 2024-11-12T14:43:59,366 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testCompactedBulkLoadedFiles Metrics about Tables on a single HBase RegionServer 2024-11-12T14:43:59,860 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:40559/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731422635009/wal.1731422635506 after 4001ms 2024-11-12T14:43:59,888 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-12T14:43:59,888 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:40559/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731422635009/wal.1731422635506 took 4029ms 2024-11-12T14:43:59,890 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:40559/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731422635009/wal.1731422635506; continuing. 2024-11-12T14:43:59,890 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:40559/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731422635009/wal.1731422635506 so closing down 2024-11-12T14:43:59,890 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-12T14:43:59,892 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000037-wal.1731422635506.temp 2024-11-12T14:43:59,921 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:40559/hbase/data/default/testReplayEditsWrittenViaHRegion/88bbcedda8de78f6ce3bcc3cde9eac6f/recovered.edits/0000000000000000037-wal.1731422635506.temp 2024-11-12T14:43:59,921 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-12T14:43:59,926 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37395 is added to blk_1073741933_1113 (size=5575) 2024-11-12T14:43:59,926 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36175 is added to blk_1073741933_1113 (size=5575) 2024-11-12T14:43:59,926 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45975 is added to blk_1073741933_1113 (size=5575) 2024-11-12T14:43:59,927 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:40559/hbase/data/default/testReplayEditsWrittenViaHRegion/88bbcedda8de78f6ce3bcc3cde9eac6f/recovered.edits/0000000000000000037-wal.1731422635506.temp (wrote 30 edits, skipped 0 edits in 0 ms) 2024-11-12T14:43:59,928 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:40559/hbase/data/default/testReplayEditsWrittenViaHRegion/88bbcedda8de78f6ce3bcc3cde9eac6f/recovered.edits/0000000000000000037-wal.1731422635506.temp to hdfs://localhost:40559/hbase/data/default/testReplayEditsWrittenViaHRegion/88bbcedda8de78f6ce3bcc3cde9eac6f/recovered.edits/0000000000000000066 2024-11-12T14:43:59,928 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 30 edits across 1 Regions in 40 ms; skipped=0; WAL=hdfs://localhost:40559/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731422635009/wal.1731422635506, size=0, length=0, corrupted=false, cancelled=false 2024-11-12T14:43:59,928 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:40559/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731422635009/wal.1731422635506, journal: Splitting hdfs://localhost:40559/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731422635009/wal.1731422635506, size=0 (0bytes) at 1731422635859Finishing writing output for hdfs://localhost:40559/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731422635009/wal.1731422635506 so closing down at 1731422639890 (+4031 ms)Creating recovered edits writer path=hdfs://localhost:40559/hbase/data/default/testReplayEditsWrittenViaHRegion/88bbcedda8de78f6ce3bcc3cde9eac6f/recovered.edits/0000000000000000037-wal.1731422635506.temp at 1731422639921 (+31 ms)3 split writer threads finished at 1731422639921Closed recovered edits writer path=hdfs://localhost:40559/hbase/data/default/testReplayEditsWrittenViaHRegion/88bbcedda8de78f6ce3bcc3cde9eac6f/recovered.edits/0000000000000000037-wal.1731422635506.temp (wrote 30 edits, skipped 0 edits in 0 ms) at 1731422639927 (+6 ms)Rename recovered edits hdfs://localhost:40559/hbase/data/default/testReplayEditsWrittenViaHRegion/88bbcedda8de78f6ce3bcc3cde9eac6f/recovered.edits/0000000000000000037-wal.1731422635506.temp to hdfs://localhost:40559/hbase/data/default/testReplayEditsWrittenViaHRegion/88bbcedda8de78f6ce3bcc3cde9eac6f/recovered.edits/0000000000000000066 at 1731422639928 (+1 ms)Processed 30 edits across 1 Regions in 40 ms; skipped=0; WAL=hdfs://localhost:40559/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731422635009/wal.1731422635506, size=0, length=0, corrupted=false, cancelled=false at 1731422639928 2024-11-12T14:43:59,929 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:40559/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731422635009/wal.1731422635506 to hdfs://localhost:40559/hbase/oldWALs/wal.1731422635506 2024-11-12T14:43:59,930 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:40559/hbase/data/default/testReplayEditsWrittenViaHRegion/88bbcedda8de78f6ce3bcc3cde9eac6f/recovered.edits/0000000000000000066 2024-11-12T14:43:59,930 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-12T14:43:59,932 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:40559/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731422635009, archiveDir=hdfs://localhost:40559/hbase/oldWALs, maxLogs=32 2024-11-12T14:43:59,946 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731422635009/wal.1731422639932, exclude list is [], retry=0 2024-11-12T14:43:59,948 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36175,DS-31e7ba0a-e37c-4fe6-8d2a-a4846cefa408,DISK] 2024-11-12T14:43:59,948 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37395,DS-7317e4d9-cef6-4b85-a947-106c3c917da9,DISK] 2024-11-12T14:43:59,949 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45975,DS-80b047b6-34d2-41c0-99c3-5cd61ca40075,DISK] 2024-11-12T14:43:59,983 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731422635009/wal.1731422639932 2024-11-12T14:43:59,983 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36027:36027),(127.0.0.1/127.0.0.1:40227:40227),(127.0.0.1/127.0.0.1:43643:43643)] 2024-11-12T14:43:59,983 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1731422635010.88bbcedda8de78f6ce3bcc3cde9eac6f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T14:43:59,985 INFO [StoreOpener-88bbcedda8de78f6ce3bcc3cde9eac6f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 88bbcedda8de78f6ce3bcc3cde9eac6f 2024-11-12T14:43:59,985 INFO [StoreOpener-88bbcedda8de78f6ce3bcc3cde9eac6f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 88bbcedda8de78f6ce3bcc3cde9eac6f columnFamilyName a 2024-11-12T14:43:59,985 DEBUG [StoreOpener-88bbcedda8de78f6ce3bcc3cde9eac6f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:43:59,990 DEBUG [StoreOpener-88bbcedda8de78f6ce3bcc3cde9eac6f-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:40559/hbase/data/default/testReplayEditsWrittenViaHRegion/88bbcedda8de78f6ce3bcc3cde9eac6f/a/1ef72724db554a86b4e868a99ce7f723 2024-11-12T14:43:59,990 INFO [StoreOpener-88bbcedda8de78f6ce3bcc3cde9eac6f-1 {}] regionserver.HStore(327): Store=88bbcedda8de78f6ce3bcc3cde9eac6f/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T14:43:59,991 INFO [StoreOpener-88bbcedda8de78f6ce3bcc3cde9eac6f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 88bbcedda8de78f6ce3bcc3cde9eac6f 2024-11-12T14:43:59,991 INFO [StoreOpener-88bbcedda8de78f6ce3bcc3cde9eac6f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 88bbcedda8de78f6ce3bcc3cde9eac6f columnFamilyName b 2024-11-12T14:43:59,991 DEBUG [StoreOpener-88bbcedda8de78f6ce3bcc3cde9eac6f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:43:59,998 DEBUG [StoreOpener-88bbcedda8de78f6ce3bcc3cde9eac6f-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:40559/hbase/data/default/testReplayEditsWrittenViaHRegion/88bbcedda8de78f6ce3bcc3cde9eac6f/b/c6d6ebb188904ad5b71917e09d882ef2 2024-11-12T14:43:59,998 INFO [StoreOpener-88bbcedda8de78f6ce3bcc3cde9eac6f-1 {}] regionserver.HStore(327): Store=88bbcedda8de78f6ce3bcc3cde9eac6f/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T14:43:59,998 INFO [StoreOpener-88bbcedda8de78f6ce3bcc3cde9eac6f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 88bbcedda8de78f6ce3bcc3cde9eac6f 2024-11-12T14:43:59,999 INFO [StoreOpener-88bbcedda8de78f6ce3bcc3cde9eac6f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 88bbcedda8de78f6ce3bcc3cde9eac6f columnFamilyName c 2024-11-12T14:43:59,999 DEBUG [StoreOpener-88bbcedda8de78f6ce3bcc3cde9eac6f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:44:00,008 DEBUG [StoreOpener-88bbcedda8de78f6ce3bcc3cde9eac6f-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:40559/hbase/data/default/testReplayEditsWrittenViaHRegion/88bbcedda8de78f6ce3bcc3cde9eac6f/c/9f3f73ddb0644a8a93a0ca2c39528ef6 2024-11-12T14:44:00,008 INFO [StoreOpener-88bbcedda8de78f6ce3bcc3cde9eac6f-1 {}] regionserver.HStore(327): Store=88bbcedda8de78f6ce3bcc3cde9eac6f/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T14:44:00,008 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 88bbcedda8de78f6ce3bcc3cde9eac6f 2024-11-12T14:44:00,009 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40559/hbase/data/default/testReplayEditsWrittenViaHRegion/88bbcedda8de78f6ce3bcc3cde9eac6f 2024-11-12T14:44:00,010 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:40559/hbase/data/default/testReplayEditsWrittenViaHRegion/88bbcedda8de78f6ce3bcc3cde9eac6f 2024-11-12T14:44:00,011 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:40559/hbase/data/default/testReplayEditsWrittenViaHRegion/88bbcedda8de78f6ce3bcc3cde9eac6f/recovered.edits/0000000000000000066 2024-11-12T14:44:00,041 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-12T14:44:00,046 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 30, skipped 0, firstSequenceIdInLog=37, maxSequenceIdInLog=66, path=hdfs://localhost:40559/hbase/data/default/testReplayEditsWrittenViaHRegion/88bbcedda8de78f6ce3bcc3cde9eac6f/recovered.edits/0000000000000000066 2024-11-12T14:44:00,046 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 88bbcedda8de78f6ce3bcc3cde9eac6f 3/3 column families, dataSize=2.55 KB heapSize=5.44 KB 2024-11-12T14:44:00,060 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40559/hbase/data/default/testReplayEditsWrittenViaHRegion/88bbcedda8de78f6ce3bcc3cde9eac6f/.tmp/a/c5687c81e1d4433b9d9196b8be7bae6d is 91, key is testReplayEditsWrittenViaHRegion/a:y0/1731422635774/Put/seqid=0 2024-11-12T14:44:00,071 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37395 is added to blk_1073741935_1115 (size=5958) 2024-11-12T14:44:00,071 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36175 is added to blk_1073741935_1115 (size=5958) 2024-11-12T14:44:00,076 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45975 is added to blk_1073741935_1115 (size=5958) 2024-11-12T14:44:00,076 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:40559/hbase/data/default/testReplayEditsWrittenViaHRegion/88bbcedda8de78f6ce3bcc3cde9eac6f/.tmp/a/c5687c81e1d4433b9d9196b8be7bae6d 2024-11-12T14:44:00,096 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40559/hbase/data/default/testReplayEditsWrittenViaHRegion/88bbcedda8de78f6ce3bcc3cde9eac6f/.tmp/b/ca71963e8fb4425895d0b1a8571e6977 is 91, key is testReplayEditsWrittenViaHRegion/b:y0/1731422635788/Put/seqid=0 2024-11-12T14:44:00,108 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36175 is added to blk_1073741936_1116 (size=5958) 2024-11-12T14:44:00,108 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37395 is added to blk_1073741936_1116 (size=5958) 2024-11-12T14:44:00,108 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45975 is added to blk_1073741936_1116 (size=5958) 2024-11-12T14:44:00,109 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:40559/hbase/data/default/testReplayEditsWrittenViaHRegion/88bbcedda8de78f6ce3bcc3cde9eac6f/.tmp/b/ca71963e8fb4425895d0b1a8571e6977 2024-11-12T14:44:00,134 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40559/hbase/data/default/testReplayEditsWrittenViaHRegion/88bbcedda8de78f6ce3bcc3cde9eac6f/.tmp/c/8258b2fe21154cd7b21e0d20759edc38 is 91, key is testReplayEditsWrittenViaHRegion/c:y0/1731422635796/Put/seqid=0 2024-11-12T14:44:00,144 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37395 is added to blk_1073741937_1117 (size=5958) 2024-11-12T14:44:00,144 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36175 is added to blk_1073741937_1117 (size=5958) 2024-11-12T14:44:00,144 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45975 is added to blk_1073741937_1117 (size=5958) 2024-11-12T14:44:00,145 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:40559/hbase/data/default/testReplayEditsWrittenViaHRegion/88bbcedda8de78f6ce3bcc3cde9eac6f/.tmp/c/8258b2fe21154cd7b21e0d20759edc38 2024-11-12T14:44:00,151 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40559/hbase/data/default/testReplayEditsWrittenViaHRegion/88bbcedda8de78f6ce3bcc3cde9eac6f/.tmp/a/c5687c81e1d4433b9d9196b8be7bae6d as hdfs://localhost:40559/hbase/data/default/testReplayEditsWrittenViaHRegion/88bbcedda8de78f6ce3bcc3cde9eac6f/a/c5687c81e1d4433b9d9196b8be7bae6d 2024-11-12T14:44:00,156 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40559/hbase/data/default/testReplayEditsWrittenViaHRegion/88bbcedda8de78f6ce3bcc3cde9eac6f/a/c5687c81e1d4433b9d9196b8be7bae6d, entries=10, sequenceid=66, filesize=5.8 K 2024-11-12T14:44:00,157 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40559/hbase/data/default/testReplayEditsWrittenViaHRegion/88bbcedda8de78f6ce3bcc3cde9eac6f/.tmp/b/ca71963e8fb4425895d0b1a8571e6977 as hdfs://localhost:40559/hbase/data/default/testReplayEditsWrittenViaHRegion/88bbcedda8de78f6ce3bcc3cde9eac6f/b/ca71963e8fb4425895d0b1a8571e6977 2024-11-12T14:44:00,162 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40559/hbase/data/default/testReplayEditsWrittenViaHRegion/88bbcedda8de78f6ce3bcc3cde9eac6f/b/ca71963e8fb4425895d0b1a8571e6977, entries=10, sequenceid=66, filesize=5.8 K 2024-11-12T14:44:00,163 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40559/hbase/data/default/testReplayEditsWrittenViaHRegion/88bbcedda8de78f6ce3bcc3cde9eac6f/.tmp/c/8258b2fe21154cd7b21e0d20759edc38 as hdfs://localhost:40559/hbase/data/default/testReplayEditsWrittenViaHRegion/88bbcedda8de78f6ce3bcc3cde9eac6f/c/8258b2fe21154cd7b21e0d20759edc38 2024-11-12T14:44:00,168 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40559/hbase/data/default/testReplayEditsWrittenViaHRegion/88bbcedda8de78f6ce3bcc3cde9eac6f/c/8258b2fe21154cd7b21e0d20759edc38, entries=10, sequenceid=66, filesize=5.8 K 2024-11-12T14:44:00,168 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~2.55 KB/2610, heapSize ~5.39 KB/5520, currentSize=0 B/0 for 88bbcedda8de78f6ce3bcc3cde9eac6f in 122ms, sequenceid=66, compaction requested=false; wal=null 2024-11-12T14:44:00,169 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:40559/hbase/data/default/testReplayEditsWrittenViaHRegion/88bbcedda8de78f6ce3bcc3cde9eac6f/recovered.edits/0000000000000000066 2024-11-12T14:44:00,170 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 88bbcedda8de78f6ce3bcc3cde9eac6f 2024-11-12T14:44:00,170 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 88bbcedda8de78f6ce3bcc3cde9eac6f 2024-11-12T14:44:00,171 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-12T14:44:00,172 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 88bbcedda8de78f6ce3bcc3cde9eac6f 2024-11-12T14:44:00,175 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40559/hbase/data/default/testReplayEditsWrittenViaHRegion/88bbcedda8de78f6ce3bcc3cde9eac6f/recovered.edits/66.seqid, newMaxSeqId=66, maxSeqId=35 2024-11-12T14:44:00,176 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 88bbcedda8de78f6ce3bcc3cde9eac6f; next sequenceid=67; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75491301, jitterRate=0.12490804493427277}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-12T14:44:00,176 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 88bbcedda8de78f6ce3bcc3cde9eac6f: Writing region info on filesystem at 1731422639983Initializing all the Stores at 1731422639984 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731422639984Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731422639984Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731422639984Obtaining lock to block concurrent updates at 1731422640046 (+62 ms)Preparing flush snapshotting stores in 88bbcedda8de78f6ce3bcc3cde9eac6f at 1731422640046Finished memstore snapshotting testReplayEditsWrittenViaHRegion,,1731422635010.88bbcedda8de78f6ce3bcc3cde9eac6f., syncing WAL and waiting on mvcc, flushsize=dataSize=2610, getHeapSize=5520, getOffHeapSize=0, getCellsCount=30 at 1731422640046Flushing stores of testReplayEditsWrittenViaHRegion,,1731422635010.88bbcedda8de78f6ce3bcc3cde9eac6f. at 1731422640046Flushing 88bbcedda8de78f6ce3bcc3cde9eac6f/a: creating writer at 1731422640046Flushing 88bbcedda8de78f6ce3bcc3cde9eac6f/a: appending metadata at 1731422640059 (+13 ms)Flushing 88bbcedda8de78f6ce3bcc3cde9eac6f/a: closing flushed file at 1731422640060 (+1 ms)Flushing 88bbcedda8de78f6ce3bcc3cde9eac6f/b: creating writer at 1731422640081 (+21 ms)Flushing 88bbcedda8de78f6ce3bcc3cde9eac6f/b: appending metadata at 1731422640096 (+15 ms)Flushing 88bbcedda8de78f6ce3bcc3cde9eac6f/b: closing flushed file at 1731422640096Flushing 88bbcedda8de78f6ce3bcc3cde9eac6f/c: creating writer at 1731422640115 (+19 ms)Flushing 88bbcedda8de78f6ce3bcc3cde9eac6f/c: appending metadata at 1731422640133 (+18 ms)Flushing 88bbcedda8de78f6ce3bcc3cde9eac6f/c: closing flushed file at 1731422640133Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6b6b7250: reopening flushed file at 1731422640150 (+17 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@27c06355: reopening flushed file at 1731422640156 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@272fe578: reopening flushed file at 1731422640163 (+7 ms)Finished flush of dataSize ~2.55 KB/2610, heapSize ~5.39 KB/5520, currentSize=0 B/0 for 88bbcedda8de78f6ce3bcc3cde9eac6f in 122ms, sequenceid=66, compaction requested=false; wal=null at 1731422640168 (+5 ms)Cleaning up temporary data from old regions at 1731422640170 (+2 ms)Region opened successfully at 1731422640176 (+6 ms) 2024-11-12T14:44:00,190 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 88bbcedda8de78f6ce3bcc3cde9eac6f, disabling compactions & flushes 2024-11-12T14:44:00,190 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1731422635010.88bbcedda8de78f6ce3bcc3cde9eac6f. 2024-11-12T14:44:00,190 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1731422635010.88bbcedda8de78f6ce3bcc3cde9eac6f. 2024-11-12T14:44:00,190 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1731422635010.88bbcedda8de78f6ce3bcc3cde9eac6f. after waiting 0 ms 2024-11-12T14:44:00,190 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1731422635010.88bbcedda8de78f6ce3bcc3cde9eac6f. 2024-11-12T14:44:00,191 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1731422635010.88bbcedda8de78f6ce3bcc3cde9eac6f. 2024-11-12T14:44:00,191 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 88bbcedda8de78f6ce3bcc3cde9eac6f: Waiting for close lock at 1731422640190Disabling compacts and flushes for region at 1731422640190Disabling writes for close at 1731422640190Writing region close event to WAL at 1731422640191 (+1 ms)Closed at 1731422640191 2024-11-12T14:44:00,194 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36175 is added to blk_1073741934_1114 (size=175) 2024-11-12T14:44:00,194 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731422635009/wal.1731422639932 not finished, retry = 0 2024-11-12T14:44:00,194 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45975 is added to blk_1073741934_1114 (size=175) 2024-11-12T14:44:00,194 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37395 is added to blk_1073741934_1114 (size=175) 2024-11-12T14:44:00,296 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-12T14:44:00,297 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1731422639932) 2024-11-12T14:44:00,313 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsWrittenViaHRegion Thread=449 (was 444) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-252052919_22 at /127.0.0.1:47856 [Waiting for operation #8] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkinstestReplayEditsWrittenViaHRegion@localhost:40559 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-28-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-252052919_22 at /127.0.0.1:50888 [Waiting for operation #15] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1685805399) connection to localhost/127.0.0.1:40559 from jenkinstestReplayEditsWrittenViaHRegion java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: AsyncFSWAL-28-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-252052919_22 at /127.0.0.1:39922 [Waiting for operation #20] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-28-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1414 (was 1348) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=348 (was 335) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=7524 (was 7752) 2024-11-12T14:44:00,313 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1414 is superior to 1024 2024-11-12T14:44:00,313 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-12T14:44:00,313 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-12T14:44:00,313 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-12T14:44:00,313 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T14:44:00,314 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T14:44:00,314 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T14:44:00,314 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-12T14:44:00,314 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=503296047, stopped=false 2024-11-12T14:44:00,315 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=3945e1ca6f47,34233,1731422599017 2024-11-12T14:44:00,317 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33387-0x100346390eb0002, quorum=127.0.0.1:55341, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-12T14:44:00,317 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33387-0x100346390eb0002, quorum=127.0.0.1:55341, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T14:44:00,318 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41431-0x100346390eb0001, quorum=127.0.0.1:55341, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-12T14:44:00,318 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41431-0x100346390eb0001, quorum=127.0.0.1:55341, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T14:44:00,319 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34233-0x100346390eb0000, quorum=127.0.0.1:55341, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-12T14:44:00,319 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34233-0x100346390eb0000, quorum=127.0.0.1:55341, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T14:44:00,319 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-12T14:44:00,319 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-12T14:44:00,319 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-12T14:44:00,320 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T14:44:00,320 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '3945e1ca6f47,41431,1731422599613' ***** 2024-11-12T14:44:00,320 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-12T14:44:00,320 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '3945e1ca6f47,33387,1731422599699' ***** 2024-11-12T14:44:00,320 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-12T14:44:00,320 INFO [RS:0;3945e1ca6f47:41431 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-12T14:44:00,320 INFO [RS:1;3945e1ca6f47:33387 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-12T14:44:00,321 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-12T14:44:00,321 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-12T14:44:00,321 INFO [RS:0;3945e1ca6f47:41431 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-12T14:44:00,321 INFO [RS:0;3945e1ca6f47:41431 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-12T14:44:00,321 INFO [RS:0;3945e1ca6f47:41431 {}] regionserver.HRegionServer(3091): Received CLOSE for 06dfc67af613562882ee0461d2aa7115 2024-11-12T14:44:00,321 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:33387-0x100346390eb0002, quorum=127.0.0.1:55341, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-12T14:44:00,321 INFO [RS:0;3945e1ca6f47:41431 {}] regionserver.HRegionServer(959): stopping server 3945e1ca6f47,41431,1731422599613 2024-11-12T14:44:00,321 INFO [RS:0;3945e1ca6f47:41431 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-12T14:44:00,322 INFO [RS:0;3945e1ca6f47:41431 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;3945e1ca6f47:41431. 2024-11-12T14:44:00,321 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 06dfc67af613562882ee0461d2aa7115, disabling compactions & flushes 2024-11-12T14:44:00,322 DEBUG [RS:0;3945e1ca6f47:41431 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-12T14:44:00,322 INFO [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1731422614808.06dfc67af613562882ee0461d2aa7115. 2024-11-12T14:44:00,322 DEBUG [RS:0;3945e1ca6f47:41431 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T14:44:00,322 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:41431-0x100346390eb0001, quorum=127.0.0.1:55341, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-12T14:44:00,322 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731422614808.06dfc67af613562882ee0461d2aa7115. 2024-11-12T14:44:00,322 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731422614808.06dfc67af613562882ee0461d2aa7115. after waiting 0 ms 2024-11-12T14:44:00,322 INFO [RS:0;3945e1ca6f47:41431 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-12T14:44:00,322 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1731422614808.06dfc67af613562882ee0461d2aa7115. 2024-11-12T14:44:00,322 DEBUG [RS:0;3945e1ca6f47:41431 {}] regionserver.HRegionServer(1325): Online Regions={06dfc67af613562882ee0461d2aa7115=testReplayEditsAfterRegionMovedWithMultiCF,,1731422614808.06dfc67af613562882ee0461d2aa7115.} 2024-11-12T14:44:00,322 DEBUG [RS:0;3945e1ca6f47:41431 {}] regionserver.HRegionServer(1351): Waiting on 06dfc67af613562882ee0461d2aa7115 2024-11-12T14:44:00,322 INFO [RS:1;3945e1ca6f47:33387 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-12T14:44:00,322 INFO [RS:1;3945e1ca6f47:33387 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-12T14:44:00,322 INFO [RS:1;3945e1ca6f47:33387 {}] regionserver.HRegionServer(959): stopping server 3945e1ca6f47,33387,1731422599699 2024-11-12T14:44:00,322 INFO [RS:1;3945e1ca6f47:33387 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-12T14:44:00,322 INFO [RS:1;3945e1ca6f47:33387 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;3945e1ca6f47:33387. 2024-11-12T14:44:00,322 DEBUG [RS:1;3945e1ca6f47:33387 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-12T14:44:00,322 DEBUG [RS:1;3945e1ca6f47:33387 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T14:44:00,322 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:34233-0x100346390eb0000, quorum=127.0.0.1:55341, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-12T14:44:00,322 INFO [RS:1;3945e1ca6f47:33387 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-12T14:44:00,322 INFO [RS:1;3945e1ca6f47:33387 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-12T14:44:00,322 INFO [RS:1;3945e1ca6f47:33387 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-12T14:44:00,322 INFO [RS:1;3945e1ca6f47:33387 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-12T14:44:00,325 INFO [RS:1;3945e1ca6f47:33387 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-12T14:44:00,325 DEBUG [RS:1;3945e1ca6f47:33387 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-12T14:44:00,325 DEBUG [RS_CLOSE_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-12T14:44:00,325 DEBUG [RS:1;3945e1ca6f47:33387 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-12T14:44:00,325 INFO [RS_CLOSE_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-12T14:44:00,325 DEBUG [RS_CLOSE_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-12T14:44:00,325 DEBUG [RS_CLOSE_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-12T14:44:00,325 DEBUG [RS_CLOSE_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-12T14:44:00,325 INFO [RS_CLOSE_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=6.86 KB heapSize=11.45 KB 2024-11-12T14:44:00,327 INFO [regionserver/3945e1ca6f47:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-12T14:44:00,328 INFO [regionserver/3945e1ca6f47:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-12T14:44:00,336 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40559/user/jenkins/test-data/41a388f8-1764-825e-e285-25964d37c9f3/data/default/testReplayEditsAfterRegionMovedWithMultiCF/06dfc67af613562882ee0461d2aa7115/recovered.edits/20.seqid, newMaxSeqId=20, maxSeqId=17 2024-11-12T14:44:00,337 INFO [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1731422614808.06dfc67af613562882ee0461d2aa7115. 2024-11-12T14:44:00,337 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 06dfc67af613562882ee0461d2aa7115: Waiting for close lock at 1731422640321Running coprocessor pre-close hooks at 1731422640321Disabling compacts and flushes for region at 1731422640321Disabling writes for close at 1731422640322 (+1 ms)Writing region close event to WAL at 1731422640332 (+10 ms)Running coprocessor post-close hooks at 1731422640337 (+5 ms)Closed at 1731422640337 2024-11-12T14:44:00,338 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1731422614808.06dfc67af613562882ee0461d2aa7115. 2024-11-12T14:44:00,352 DEBUG [RS_CLOSE_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40559/user/jenkins/test-data/41a388f8-1764-825e-e285-25964d37c9f3/data/hbase/meta/1588230740/.tmp/info/2f71c8e3d0944839b2f21b00d6f9f90a is 205, key is testReplayEditsAfterRegionMovedWithMultiCF,,1731422614808.06dfc67af613562882ee0461d2aa7115./info:regioninfo/1731422617990/Put/seqid=0 2024-11-12T14:44:00,363 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37395 is added to blk_1073741938_1118 (size=8243) 2024-11-12T14:44:00,363 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36175 is added to blk_1073741938_1118 (size=8243) 2024-11-12T14:44:00,363 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45975 is added to blk_1073741938_1118 (size=8243) 2024-11-12T14:44:00,364 INFO [RS_CLOSE_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.65 KB at sequenceid=21 (bloomFilter=true), to=hdfs://localhost:40559/user/jenkins/test-data/41a388f8-1764-825e-e285-25964d37c9f3/data/hbase/meta/1588230740/.tmp/info/2f71c8e3d0944839b2f21b00d6f9f90a 2024-11-12T14:44:00,384 DEBUG [RS_CLOSE_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40559/user/jenkins/test-data/41a388f8-1764-825e-e285-25964d37c9f3/data/hbase/meta/1588230740/.tmp/ns/c2a164001b204cb88acc9c086776f1a4 is 43, key is default/ns:d/1731422601652/Put/seqid=0 2024-11-12T14:44:00,399 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37395 is added to blk_1073741939_1119 (size=5153) 2024-11-12T14:44:00,399 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45975 is added to blk_1073741939_1119 (size=5153) 2024-11-12T14:44:00,400 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36175 is added to blk_1073741939_1119 (size=5153) 2024-11-12T14:44:00,401 INFO [RS_CLOSE_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=21 (bloomFilter=true), to=hdfs://localhost:40559/user/jenkins/test-data/41a388f8-1764-825e-e285-25964d37c9f3/data/hbase/meta/1588230740/.tmp/ns/c2a164001b204cb88acc9c086776f1a4 2024-11-12T14:44:00,423 DEBUG [RS_CLOSE_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40559/user/jenkins/test-data/41a388f8-1764-825e-e285-25964d37c9f3/data/hbase/meta/1588230740/.tmp/table/ad2dba0d7b0045c0bcaadf4129dd5dc5 is 78, key is testReplayEditsAfterRegionMovedWithMultiCF/table:state/1731422615228/Put/seqid=0 2024-11-12T14:44:00,429 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37395 is added to blk_1073741940_1120 (size=5431) 2024-11-12T14:44:00,429 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45975 is added to blk_1073741940_1120 (size=5431) 2024-11-12T14:44:00,429 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36175 is added to blk_1073741940_1120 (size=5431) 2024-11-12T14:44:00,429 INFO [RS_CLOSE_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=148 B at sequenceid=21 (bloomFilter=true), to=hdfs://localhost:40559/user/jenkins/test-data/41a388f8-1764-825e-e285-25964d37c9f3/data/hbase/meta/1588230740/.tmp/table/ad2dba0d7b0045c0bcaadf4129dd5dc5 2024-11-12T14:44:00,435 DEBUG [RS_CLOSE_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40559/user/jenkins/test-data/41a388f8-1764-825e-e285-25964d37c9f3/data/hbase/meta/1588230740/.tmp/info/2f71c8e3d0944839b2f21b00d6f9f90a as hdfs://localhost:40559/user/jenkins/test-data/41a388f8-1764-825e-e285-25964d37c9f3/data/hbase/meta/1588230740/info/2f71c8e3d0944839b2f21b00d6f9f90a 2024-11-12T14:44:00,444 INFO [RS_CLOSE_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40559/user/jenkins/test-data/41a388f8-1764-825e-e285-25964d37c9f3/data/hbase/meta/1588230740/info/2f71c8e3d0944839b2f21b00d6f9f90a, entries=18, sequenceid=21, filesize=8.0 K 2024-11-12T14:44:00,445 DEBUG [RS_CLOSE_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40559/user/jenkins/test-data/41a388f8-1764-825e-e285-25964d37c9f3/data/hbase/meta/1588230740/.tmp/ns/c2a164001b204cb88acc9c086776f1a4 as hdfs://localhost:40559/user/jenkins/test-data/41a388f8-1764-825e-e285-25964d37c9f3/data/hbase/meta/1588230740/ns/c2a164001b204cb88acc9c086776f1a4 2024-11-12T14:44:00,452 INFO [RS_CLOSE_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40559/user/jenkins/test-data/41a388f8-1764-825e-e285-25964d37c9f3/data/hbase/meta/1588230740/ns/c2a164001b204cb88acc9c086776f1a4, entries=2, sequenceid=21, filesize=5.0 K 2024-11-12T14:44:00,453 DEBUG [RS_CLOSE_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40559/user/jenkins/test-data/41a388f8-1764-825e-e285-25964d37c9f3/data/hbase/meta/1588230740/.tmp/table/ad2dba0d7b0045c0bcaadf4129dd5dc5 as hdfs://localhost:40559/user/jenkins/test-data/41a388f8-1764-825e-e285-25964d37c9f3/data/hbase/meta/1588230740/table/ad2dba0d7b0045c0bcaadf4129dd5dc5 2024-11-12T14:44:00,461 INFO [RS_CLOSE_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40559/user/jenkins/test-data/41a388f8-1764-825e-e285-25964d37c9f3/data/hbase/meta/1588230740/table/ad2dba0d7b0045c0bcaadf4129dd5dc5, entries=2, sequenceid=21, filesize=5.3 K 2024-11-12T14:44:00,465 INFO [RS_CLOSE_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~6.86 KB/7029, heapSize ~11.16 KB/11424, currentSize=0 B/0 for 1588230740 in 140ms, sequenceid=21, compaction requested=false 2024-11-12T14:44:00,472 DEBUG [RS_CLOSE_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40559/user/jenkins/test-data/41a388f8-1764-825e-e285-25964d37c9f3/data/hbase/meta/1588230740/recovered.edits/24.seqid, newMaxSeqId=24, maxSeqId=1 2024-11-12T14:44:00,472 DEBUG [RS_CLOSE_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-12T14:44:00,472 INFO [RS_CLOSE_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-12T14:44:00,472 DEBUG [RS_CLOSE_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731422640325Running coprocessor pre-close hooks at 1731422640325Disabling compacts and flushes for region at 1731422640325Disabling writes for close at 1731422640325Obtaining lock to block concurrent updates at 1731422640325Preparing flush snapshotting stores in 1588230740 at 1731422640325Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=7029, getHeapSize=11664, getOffHeapSize=0, getCellsCount=48 at 1731422640326 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1731422640326Flushing 1588230740/info: creating writer at 1731422640327 (+1 ms)Flushing 1588230740/info: appending metadata at 1731422640351 (+24 ms)Flushing 1588230740/info: closing flushed file at 1731422640351Flushing 1588230740/ns: creating writer at 1731422640369 (+18 ms)Flushing 1588230740/ns: appending metadata at 1731422640384 (+15 ms)Flushing 1588230740/ns: closing flushed file at 1731422640384Flushing 1588230740/table: creating writer at 1731422640406 (+22 ms)Flushing 1588230740/table: appending metadata at 1731422640422 (+16 ms)Flushing 1588230740/table: closing flushed file at 1731422640422Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4b36d297: reopening flushed file at 1731422640434 (+12 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@19fbfe43: reopening flushed file at 1731422640444 (+10 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@574c8921: reopening flushed file at 1731422640452 (+8 ms)Finished flush of dataSize ~6.86 KB/7029, heapSize ~11.16 KB/11424, currentSize=0 B/0 for 1588230740 in 140ms, sequenceid=21, compaction requested=false at 1731422640465 (+13 ms)Writing region close event to WAL at 1731422640468 (+3 ms)Running coprocessor post-close hooks at 1731422640472 (+4 ms)Closed at 1731422640472 2024-11-12T14:44:00,473 DEBUG [RS_CLOSE_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-12T14:44:00,522 INFO [RS:0;3945e1ca6f47:41431 {}] regionserver.HRegionServer(976): stopping server 3945e1ca6f47,41431,1731422599613; all regions closed. 2024-11-12T14:44:00,525 INFO [RS:1;3945e1ca6f47:33387 {}] regionserver.HRegionServer(976): stopping server 3945e1ca6f47,33387,1731422599699; all regions closed. 2024-11-12T14:44:00,533 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45975 is added to blk_1073741834_1010 (size=3621) 2024-11-12T14:44:00,534 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37395 is added to blk_1073741834_1010 (size=3621) 2024-11-12T14:44:00,534 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36175 is added to blk_1073741834_1010 (size=3621) 2024-11-12T14:44:00,540 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37395 is added to blk_1073741836_1012 (size=9487) 2024-11-12T14:44:00,540 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/41a388f8-1764-825e-e285-25964d37c9f3/WALs/3945e1ca6f47,33387,1731422599699/3945e1ca6f47%2C33387%2C1731422599699.meta.1731422601484.meta not finished, retry = 0 2024-11-12T14:44:00,540 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45975 is added to blk_1073741836_1012 (size=9487) 2024-11-12T14:44:00,540 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36175 is added to blk_1073741836_1012 (size=9487) 2024-11-12T14:44:00,541 DEBUG [RS:0;3945e1ca6f47:41431 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/41a388f8-1764-825e-e285-25964d37c9f3/oldWALs 2024-11-12T14:44:00,541 INFO [RS:0;3945e1ca6f47:41431 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 3945e1ca6f47%2C41431%2C1731422599613:(num 1731422601050) 2024-11-12T14:44:00,541 DEBUG [RS:0;3945e1ca6f47:41431 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T14:44:00,541 INFO [RS:0;3945e1ca6f47:41431 {}] regionserver.LeaseManager(133): Closed leases 2024-11-12T14:44:00,541 INFO [RS:0;3945e1ca6f47:41431 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-12T14:44:00,542 INFO [RS:0;3945e1ca6f47:41431 {}] hbase.ChoreService(370): Chore service for: regionserver/3945e1ca6f47:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-12T14:44:00,542 INFO [RS:0;3945e1ca6f47:41431 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-12T14:44:00,542 INFO [RS:0;3945e1ca6f47:41431 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-12T14:44:00,542 INFO [RS:0;3945e1ca6f47:41431 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-12T14:44:00,542 INFO [RS:0;3945e1ca6f47:41431 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-12T14:44:00,542 INFO [RS:0;3945e1ca6f47:41431 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:41431 2024-11-12T14:44:00,542 INFO [regionserver/3945e1ca6f47:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-12T14:44:00,544 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41431-0x100346390eb0001, quorum=127.0.0.1:55341, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/3945e1ca6f47,41431,1731422599613 2024-11-12T14:44:00,544 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34233-0x100346390eb0000, quorum=127.0.0.1:55341, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-12T14:44:00,544 INFO [RS:0;3945e1ca6f47:41431 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-12T14:44:00,545 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [3945e1ca6f47,41431,1731422599613] 2024-11-12T14:44:00,546 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/3945e1ca6f47,41431,1731422599613 already deleted, retry=false 2024-11-12T14:44:00,546 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 3945e1ca6f47,41431,1731422599613 expired; onlineServers=1 2024-11-12T14:44:00,643 DEBUG [RS:1;3945e1ca6f47:33387 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/41a388f8-1764-825e-e285-25964d37c9f3/oldWALs 2024-11-12T14:44:00,643 INFO [RS:1;3945e1ca6f47:33387 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 3945e1ca6f47%2C33387%2C1731422599699.meta:.meta(num 1731422601484) 2024-11-12T14:44:00,645 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41431-0x100346390eb0001, quorum=127.0.0.1:55341, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-12T14:44:00,645 INFO [RS:0;3945e1ca6f47:41431 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-12T14:44:00,645 INFO [RS:0;3945e1ca6f47:41431 {}] regionserver.HRegionServer(1031): Exiting; stopping=3945e1ca6f47,41431,1731422599613; zookeeper connection closed. 2024-11-12T14:44:00,645 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41431-0x100346390eb0001, quorum=127.0.0.1:55341, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-12T14:44:00,645 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@72703307 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@72703307 2024-11-12T14:44:00,645 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37395 is added to blk_1073741833_1009 (size=175) 2024-11-12T14:44:00,646 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36175 is added to blk_1073741833_1009 (size=175) 2024-11-12T14:44:00,646 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45975 is added to blk_1073741833_1009 (size=175) 2024-11-12T14:44:00,648 DEBUG [RS:1;3945e1ca6f47:33387 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/41a388f8-1764-825e-e285-25964d37c9f3/oldWALs 2024-11-12T14:44:00,648 INFO [RS:1;3945e1ca6f47:33387 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 3945e1ca6f47%2C33387%2C1731422599699:(num 1731422601049) 2024-11-12T14:44:00,648 DEBUG [RS:1;3945e1ca6f47:33387 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T14:44:00,648 INFO [RS:1;3945e1ca6f47:33387 {}] regionserver.LeaseManager(133): Closed leases 2024-11-12T14:44:00,648 INFO [RS:1;3945e1ca6f47:33387 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-12T14:44:00,648 INFO [RS:1;3945e1ca6f47:33387 {}] hbase.ChoreService(370): Chore service for: regionserver/3945e1ca6f47:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-12T14:44:00,649 INFO [RS:1;3945e1ca6f47:33387 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-12T14:44:00,649 INFO [regionserver/3945e1ca6f47:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-12T14:44:00,649 INFO [RS:1;3945e1ca6f47:33387 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:33387 2024-11-12T14:44:00,650 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33387-0x100346390eb0002, quorum=127.0.0.1:55341, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/3945e1ca6f47,33387,1731422599699 2024-11-12T14:44:00,650 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34233-0x100346390eb0000, quorum=127.0.0.1:55341, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-12T14:44:00,650 INFO [RS:1;3945e1ca6f47:33387 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-12T14:44:00,651 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [3945e1ca6f47,33387,1731422599699] 2024-11-12T14:44:00,652 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/3945e1ca6f47,33387,1731422599699 already deleted, retry=false 2024-11-12T14:44:00,652 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 3945e1ca6f47,33387,1731422599699 expired; onlineServers=0 2024-11-12T14:44:00,652 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '3945e1ca6f47,34233,1731422599017' ***** 2024-11-12T14:44:00,652 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-12T14:44:00,652 INFO [M:0;3945e1ca6f47:34233 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-12T14:44:00,652 INFO [M:0;3945e1ca6f47:34233 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-12T14:44:00,652 DEBUG [M:0;3945e1ca6f47:34233 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-12T14:44:00,652 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-12T14:44:00,652 DEBUG [M:0;3945e1ca6f47:34233 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-12T14:44:00,652 DEBUG [master/3945e1ca6f47:0:becomeActiveMaster-HFileCleaner.small.0-1731422600704 {}] cleaner.HFileCleaner(306): Exit Thread[master/3945e1ca6f47:0:becomeActiveMaster-HFileCleaner.small.0-1731422600704,5,FailOnTimeoutGroup] 2024-11-12T14:44:00,652 DEBUG [master/3945e1ca6f47:0:becomeActiveMaster-HFileCleaner.large.0-1731422600701 {}] cleaner.HFileCleaner(306): Exit Thread[master/3945e1ca6f47:0:becomeActiveMaster-HFileCleaner.large.0-1731422600701,5,FailOnTimeoutGroup] 2024-11-12T14:44:00,653 INFO [M:0;3945e1ca6f47:34233 {}] hbase.ChoreService(370): Chore service for: master/3945e1ca6f47:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-12T14:44:00,653 INFO [M:0;3945e1ca6f47:34233 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-12T14:44:00,653 DEBUG [M:0;3945e1ca6f47:34233 {}] master.HMaster(1795): Stopping service threads 2024-11-12T14:44:00,653 INFO [M:0;3945e1ca6f47:34233 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-12T14:44:00,653 INFO [M:0;3945e1ca6f47:34233 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-12T14:44:00,653 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34233-0x100346390eb0000, quorum=127.0.0.1:55341, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-12T14:44:00,654 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34233-0x100346390eb0000, quorum=127.0.0.1:55341, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T14:44:00,654 INFO [M:0;3945e1ca6f47:34233 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-12T14:44:00,654 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-12T14:44:00,654 DEBUG [M:0;3945e1ca6f47:34233 {}] zookeeper.ZKUtil(347): master:34233-0x100346390eb0000, quorum=127.0.0.1:55341, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-12T14:44:00,654 WARN [M:0;3945e1ca6f47:34233 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-12T14:44:00,655 INFO [M:0;3945e1ca6f47:34233 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:40559/user/jenkins/test-data/41a388f8-1764-825e-e285-25964d37c9f3/.lastflushedseqids 2024-11-12T14:44:00,668 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37395 is added to blk_1073741941_1121 (size=138) 2024-11-12T14:44:00,668 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45975 is added to blk_1073741941_1121 (size=138) 2024-11-12T14:44:00,668 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36175 is added to blk_1073741941_1121 (size=138) 2024-11-12T14:44:00,669 INFO [M:0;3945e1ca6f47:34233 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-12T14:44:00,669 INFO [M:0;3945e1ca6f47:34233 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-12T14:44:00,669 DEBUG [M:0;3945e1ca6f47:34233 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-12T14:44:00,669 INFO [M:0;3945e1ca6f47:34233 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-12T14:44:00,669 DEBUG [M:0;3945e1ca6f47:34233 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-12T14:44:00,669 DEBUG [M:0;3945e1ca6f47:34233 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-12T14:44:00,669 DEBUG [M:0;3945e1ca6f47:34233 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-12T14:44:00,669 INFO [M:0;3945e1ca6f47:34233 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=68.34 KB heapSize=83.73 KB 2024-11-12T14:44:00,686 DEBUG [M:0;3945e1ca6f47:34233 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40559/user/jenkins/test-data/41a388f8-1764-825e-e285-25964d37c9f3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/18922fdb69de40fcb926d321665dcfe1 is 82, key is hbase:meta,,1/info:regioninfo/1731422601591/Put/seqid=0 2024-11-12T14:44:00,694 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37395 is added to blk_1073741942_1122 (size=5672) 2024-11-12T14:44:00,694 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45975 is added to blk_1073741942_1122 (size=5672) 2024-11-12T14:44:00,694 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36175 is added to blk_1073741942_1122 (size=5672) 2024-11-12T14:44:00,695 INFO [M:0;3945e1ca6f47:34233 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:40559/user/jenkins/test-data/41a388f8-1764-825e-e285-25964d37c9f3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/18922fdb69de40fcb926d321665dcfe1 2024-11-12T14:44:00,719 DEBUG [M:0;3945e1ca6f47:34233 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40559/user/jenkins/test-data/41a388f8-1764-825e-e285-25964d37c9f3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/88e9f45a20344e598732a6014d8802f6 is 1076, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1731422615233/Put/seqid=0 2024-11-12T14:44:00,724 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45975 is added to blk_1073741943_1123 (size=7755) 2024-11-12T14:44:00,724 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36175 is added to blk_1073741943_1123 (size=7755) 2024-11-12T14:44:00,725 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37395 is added to blk_1073741943_1123 (size=7755) 2024-11-12T14:44:00,725 INFO [M:0;3945e1ca6f47:34233 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=67.61 KB at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:40559/user/jenkins/test-data/41a388f8-1764-825e-e285-25964d37c9f3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/88e9f45a20344e598732a6014d8802f6 2024-11-12T14:44:00,730 INFO [M:0;3945e1ca6f47:34233 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 88e9f45a20344e598732a6014d8802f6 2024-11-12T14:44:00,746 DEBUG [M:0;3945e1ca6f47:34233 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40559/user/jenkins/test-data/41a388f8-1764-825e-e285-25964d37c9f3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/5f69863644ce425394441719589a827e is 69, key is 3945e1ca6f47,33387,1731422599699/rs:state/1731422600808/Put/seqid=0 2024-11-12T14:44:00,751 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33387-0x100346390eb0002, quorum=127.0.0.1:55341, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-12T14:44:00,751 INFO [RS:1;3945e1ca6f47:33387 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-12T14:44:00,751 INFO [RS:1;3945e1ca6f47:33387 {}] regionserver.HRegionServer(1031): Exiting; stopping=3945e1ca6f47,33387,1731422599699; zookeeper connection closed. 2024-11-12T14:44:00,751 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33387-0x100346390eb0002, quorum=127.0.0.1:55341, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-12T14:44:00,751 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@4932d52b {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@4932d52b 2024-11-12T14:44:00,752 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 3 regionserver(s) complete 2024-11-12T14:44:00,759 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45975 is added to blk_1073741944_1124 (size=5445) 2024-11-12T14:44:00,759 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37395 is added to blk_1073741944_1124 (size=5445) 2024-11-12T14:44:00,759 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36175 is added to blk_1073741944_1124 (size=5445) 2024-11-12T14:44:00,760 INFO [M:0;3945e1ca6f47:34233 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=249 B at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:40559/user/jenkins/test-data/41a388f8-1764-825e-e285-25964d37c9f3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/5f69863644ce425394441719589a827e 2024-11-12T14:44:00,764 INFO [M:0;3945e1ca6f47:34233 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 5f69863644ce425394441719589a827e 2024-11-12T14:44:00,765 DEBUG [M:0;3945e1ca6f47:34233 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40559/user/jenkins/test-data/41a388f8-1764-825e-e285-25964d37c9f3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/18922fdb69de40fcb926d321665dcfe1 as hdfs://localhost:40559/user/jenkins/test-data/41a388f8-1764-825e-e285-25964d37c9f3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/18922fdb69de40fcb926d321665dcfe1 2024-11-12T14:44:00,770 INFO [M:0;3945e1ca6f47:34233 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40559/user/jenkins/test-data/41a388f8-1764-825e-e285-25964d37c9f3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/18922fdb69de40fcb926d321665dcfe1, entries=8, sequenceid=168, filesize=5.5 K 2024-11-12T14:44:00,771 DEBUG [M:0;3945e1ca6f47:34233 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40559/user/jenkins/test-data/41a388f8-1764-825e-e285-25964d37c9f3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/88e9f45a20344e598732a6014d8802f6 as hdfs://localhost:40559/user/jenkins/test-data/41a388f8-1764-825e-e285-25964d37c9f3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/88e9f45a20344e598732a6014d8802f6 2024-11-12T14:44:00,776 INFO [M:0;3945e1ca6f47:34233 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 88e9f45a20344e598732a6014d8802f6 2024-11-12T14:44:00,776 INFO [M:0;3945e1ca6f47:34233 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40559/user/jenkins/test-data/41a388f8-1764-825e-e285-25964d37c9f3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/88e9f45a20344e598732a6014d8802f6, entries=17, sequenceid=168, filesize=7.6 K 2024-11-12T14:44:00,777 DEBUG [M:0;3945e1ca6f47:34233 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40559/user/jenkins/test-data/41a388f8-1764-825e-e285-25964d37c9f3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/5f69863644ce425394441719589a827e as hdfs://localhost:40559/user/jenkins/test-data/41a388f8-1764-825e-e285-25964d37c9f3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/5f69863644ce425394441719589a827e 2024-11-12T14:44:00,782 INFO [M:0;3945e1ca6f47:34233 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 5f69863644ce425394441719589a827e 2024-11-12T14:44:00,782 INFO [M:0;3945e1ca6f47:34233 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40559/user/jenkins/test-data/41a388f8-1764-825e-e285-25964d37c9f3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/5f69863644ce425394441719589a827e, entries=3, sequenceid=168, filesize=5.3 K 2024-11-12T14:44:00,784 INFO [M:0;3945e1ca6f47:34233 {}] regionserver.HRegion(3140): Finished flush of dataSize ~68.34 KB/69984, heapSize ~83.43 KB/85432, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 115ms, sequenceid=168, compaction requested=false 2024-11-12T14:44:00,785 INFO [M:0;3945e1ca6f47:34233 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-12T14:44:00,785 DEBUG [M:0;3945e1ca6f47:34233 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731422640669Disabling compacts and flushes for region at 1731422640669Disabling writes for close at 1731422640669Obtaining lock to block concurrent updates at 1731422640670 (+1 ms)Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731422640670Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=69984, getHeapSize=85672, getOffHeapSize=0, getCellsCount=195 at 1731422640670Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731422640670Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731422640670Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731422640686 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731422640686Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731422640700 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731422640719 (+19 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731422640719Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731422640730 (+11 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731422640746 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731422640746Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1c56826: reopening flushed file at 1731422640764 (+18 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@372fa305: reopening flushed file at 1731422640770 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@65c06bbc: reopening flushed file at 1731422640777 (+7 ms)Finished flush of dataSize ~68.34 KB/69984, heapSize ~83.43 KB/85432, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 115ms, sequenceid=168, compaction requested=false at 1731422640784 (+7 ms)Writing region close event to WAL at 1731422640785 (+1 ms)Closed at 1731422640785 2024-11-12T14:44:00,787 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45975 is added to blk_1073741830_1006 (size=83606) 2024-11-12T14:44:00,787 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36175 is added to blk_1073741830_1006 (size=83606) 2024-11-12T14:44:00,787 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37395 is added to blk_1073741830_1006 (size=83606) 2024-11-12T14:44:00,789 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-12T14:44:00,789 INFO [M:0;3945e1ca6f47:34233 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-12T14:44:00,789 INFO [M:0;3945e1ca6f47:34233 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:34233 2024-11-12T14:44:00,789 INFO [M:0;3945e1ca6f47:34233 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-12T14:44:00,890 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34233-0x100346390eb0000, quorum=127.0.0.1:55341, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-12T14:44:00,890 INFO [M:0;3945e1ca6f47:34233 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-12T14:44:00,890 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34233-0x100346390eb0000, quorum=127.0.0.1:55341, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-12T14:44:00,897 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731422618341/wal.1731422619019 with renewLeaseKey: DEFAULT_16655 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731422618341/wal.1731422619019 (inode 16655) Holder DFSClient_NONMAPREDUCE_-425049348_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731422618341/wal.1731422619019 (inode 16655) Holder DFSClient_NONMAPREDUCE_-425049348_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy45.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy46.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 25 more 2024-11-12T14:44:00,898 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731422610194/wal.1731422610422 with renewLeaseKey: DEFAULT_16586 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:44:00,898 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731422635009/wal.1731422635506 with renewLeaseKey: DEFAULT_16767 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:44:00,901 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731422619271/wal.1731422628822 with renewLeaseKey: DEFAULT_16678 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731422619271/wal.1731422628822 (inode 16678) Holder DFSClient_NONMAPREDUCE_-425049348_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731422619271/wal.1731422628822 (inode 16678) Holder DFSClient_NONMAPREDUCE_-425049348_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy45.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy46.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 25 more 2024-11-12T14:44:00,901 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731422603139/wal.1731422603229 with renewLeaseKey: DEFAULT_16506 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:44:00,904 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testdatalosswheninputerror-manual,16010,1731422629155/wal.1731422629947 with renewLeaseKey: DEFAULT_16704 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testdatalosswheninputerror-manual,16010,1731422629155/wal.1731422629947 (inode 16704) Holder DFSClient_NONMAPREDUCE_-425049348_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testdatalosswheninputerror-manual,16010,1731422629155/wal.1731422629947 (inode 16704) Holder DFSClient_NONMAPREDUCE_-425049348_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy45.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy46.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 25 more 2024-11-12T14:44:00,904 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731422630179/wal.1731422630314 with renewLeaseKey: DEFAULT_16726 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:44:00,907 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testnameconflictwhensplit1-manual,16010,1731422602580/wal.1731422602956 with renewLeaseKey: DEFAULT_16485 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testnameconflictwhensplit1-manual,16010,1731422602580/wal.1731422602956 (inode 16485) Holder DFSClient_NONMAPREDUCE_-425049348_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testnameconflictwhensplit1-manual,16010,1731422602580/wal.1731422602956 (inode 16485) Holder DFSClient_NONMAPREDUCE_-425049348_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy45.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy46.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 25 more 2024-11-12T14:44:00,909 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testnameconflictwhensplit0-manual,16010,1731422601865/wal.1731422602287 with renewLeaseKey: DEFAULT_16462 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testnameconflictwhensplit0-manual,16010,1731422601865/wal.1731422602287 (inode 16462) Holder DFSClient_NONMAPREDUCE_-425049348_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testnameconflictwhensplit0-manual,16010,1731422601865/wal.1731422602287 (inode 16462) Holder DFSClient_NONMAPREDUCE_-425049348_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy45.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy46.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 25 more 2024-11-12T14:44:00,915 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@b7f81b{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-12T14:44:00,917 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@5d3aa315{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-12T14:44:00,917 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-12T14:44:00,917 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@606c1a1{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-12T14:44:00,918 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4d7caafc{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5a5e6c5c-3b82-c3b3-1527-3525a941b154/hadoop.log.dir/,STOPPED} 2024-11-12T14:44:00,921 WARN [BP-1980882744-172.17.0.2-1731422596004 heartbeating to localhost/127.0.0.1:40559 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-12T14:44:00,921 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-12T14:44:00,921 WARN [BP-1980882744-172.17.0.2-1731422596004 heartbeating to localhost/127.0.0.1:40559 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1980882744-172.17.0.2-1731422596004 (Datanode Uuid b4a8cc10-901f-4240-88db-9ed40bfbd283) service to localhost/127.0.0.1:40559 2024-11-12T14:44:00,921 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-12T14:44:00,922 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5a5e6c5c-3b82-c3b3-1527-3525a941b154/cluster_22b4e46d-f388-f2af-b32c-f9fd886af812/data/data5/current/BP-1980882744-172.17.0.2-1731422596004 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-12T14:44:00,923 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5a5e6c5c-3b82-c3b3-1527-3525a941b154/cluster_22b4e46d-f388-f2af-b32c-f9fd886af812/data/data6/current/BP-1980882744-172.17.0.2-1731422596004 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-12T14:44:00,923 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-12T14:44:00,930 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@565a2c18{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-12T14:44:00,931 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@311fb016{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-12T14:44:00,931 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-12T14:44:00,931 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@540114bd{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-12T14:44:00,932 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3fe9d746{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5a5e6c5c-3b82-c3b3-1527-3525a941b154/hadoop.log.dir/,STOPPED} 2024-11-12T14:44:00,934 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-12T14:44:00,934 WARN [BP-1980882744-172.17.0.2-1731422596004 heartbeating to localhost/127.0.0.1:40559 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-12T14:44:00,934 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-12T14:44:00,934 WARN [BP-1980882744-172.17.0.2-1731422596004 heartbeating to localhost/127.0.0.1:40559 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1980882744-172.17.0.2-1731422596004 (Datanode Uuid 46e43ae4-d140-41eb-8f46-d79b5d55189d) service to localhost/127.0.0.1:40559 2024-11-12T14:44:00,935 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5a5e6c5c-3b82-c3b3-1527-3525a941b154/cluster_22b4e46d-f388-f2af-b32c-f9fd886af812/data/data3/current/BP-1980882744-172.17.0.2-1731422596004 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-12T14:44:00,935 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5a5e6c5c-3b82-c3b3-1527-3525a941b154/cluster_22b4e46d-f388-f2af-b32c-f9fd886af812/data/data4/current/BP-1980882744-172.17.0.2-1731422596004 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-12T14:44:00,935 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-12T14:44:00,948 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@388bacc3{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-12T14:44:00,949 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@6f48fbbe{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-12T14:44:00,949 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-12T14:44:00,949 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@63c29ae0{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-12T14:44:00,949 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3f494b95{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5a5e6c5c-3b82-c3b3-1527-3525a941b154/hadoop.log.dir/,STOPPED} 2024-11-12T14:44:00,950 WARN [BP-1980882744-172.17.0.2-1731422596004 heartbeating to localhost/127.0.0.1:40559 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-12T14:44:00,950 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-12T14:44:00,950 WARN [BP-1980882744-172.17.0.2-1731422596004 heartbeating to localhost/127.0.0.1:40559 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1980882744-172.17.0.2-1731422596004 (Datanode Uuid 9aab8b87-083c-4077-8d4b-86455c74734f) service to localhost/127.0.0.1:40559 2024-11-12T14:44:00,950 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-12T14:44:00,951 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5a5e6c5c-3b82-c3b3-1527-3525a941b154/cluster_22b4e46d-f388-f2af-b32c-f9fd886af812/data/data1/current/BP-1980882744-172.17.0.2-1731422596004 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-12T14:44:00,952 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5a5e6c5c-3b82-c3b3-1527-3525a941b154/cluster_22b4e46d-f388-f2af-b32c-f9fd886af812/data/data2/current/BP-1980882744-172.17.0.2-1731422596004 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-12T14:44:00,952 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-12T14:44:00,960 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@6878e65d{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-12T14:44:00,960 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@2f30922e{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-12T14:44:00,960 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-12T14:44:00,961 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4e578b56{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-12T14:44:00,961 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6bdd7391{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5a5e6c5c-3b82-c3b3-1527-3525a941b154/hadoop.log.dir/,STOPPED} 2024-11-12T14:44:00,973 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-12T14:44:01,039 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down